Page Comparison

cat > make_repo.sh <<EOT
#!/bin/bash

#DEP_FILE=""
DEP_FILE="req.txt"
LOG_FILE=""
#LOG_FILE="errors/data/repo/repo.log"
function print_log {
  if [[ "\$LOG_FILE" != "" ]]; then
    echo "\$1" >> \$LOG_FILE
  fi
}

## Loop for get build components dependencies
echo "" > list_to_check
echo "publicsuffix" > final_list
sed -i '/^[ \\t]*\$/d' list_component
echo "" >> list_component
echo "" > temp
while read comp; do
  ## Skip empty line
  if [[ "" == "\$comp" ]]; then
    continue
  fi 
  echo -e "\\033[33m =====> Getting depends for \$comp... \\033[0m"
  res=\$(apt-rdepends --build-depends --follow=DEPENDS \$comp)
  echo "\$res" | grep 'Build-Depends:' > tmp
  echo "\$res" | grep 'Build-Depends-Indep:' >> tmp
  sed 's/^[ \\t]*Build-Depends:[ \\t]*//; s/^[ \\t]*Build-Depends-Indep:[ \\t]*//' tmp | sed 's/[ \\t]*(.*)[ \\t]*\$//' >> temp
  rm -f tmp
  if [[ "\$DEP_FILE" != "" ]]; then 
    sed "s/^/\$comp:/g" temp >> \$DEP_FILE
  fi
done < list_component

## Preparation of initial list of dependencies
sort -u temp > list_to_check
sed -i '/^[ \\t]*\$/d' list_to_check
end=\$(cat list_to_check)
echo "" >> list_to_check

mkdir /data/repo/pkg/_tdir
apt-get -R /data/repo/pkg/_tdir update

## Loop for get packages dependencies
while [[ "\$end" != "" ]]; do
  echo "" > temp
  while read comp; do
    ## Skip empty line
    if [[ "" == "\$comp" ]]; then
      continue
    fi
    ## Skip packages that are already checked for dependencies
    res=\$(grep "^[ \\t]*\$comp[ \\t]*\\\$" final_list)
    if [[ "" != "\$res" ]]; then
      continue
    fi
    echo "=====> Getting depends for \$comp..."
    apt-get -R /data/repo/pkg/_tdir install --dry-run \$comp | grep "^Inst " | sed 's/^Inst //; s/ .*\$//' >> temp
    ##apt-cache depends --no-pre-depends --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances apt-cache depends --no-pre-depends --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances \$comp | sed 's/^[ \t|]*Depends: //; /</d; s/^[ \\t]*//' | sed 's/^[ \\t|]*Depends: //; /</d; s/^[ \\t]* final_list)
    if [[ "" != "\$res" ]]; then
      continue
    fi
    echo "=====> Getting depends for \$comp..."
    apt-get -R /data/repo/pkg/_tdir install --dry-run \$comp | grep "^Inst " | sed 's/^Inst //; s/ .*\$//' >> temp
    ## Add checked package to the list of already checked packages
    echo "\$comp" >> final_list
    if [[ "\$DEP_FILE" != "" ]]; then
      sed "s/^/\$comp:/g" temp >> \$DEP_FILE
    fi
  done < list_to_check

  ## Remove repeated records from the list
  sort -u temp > list_to_check
  ## Remove empty lines if any
  sed -i '/^[ \\t]*\$/d' list_to_check
  sed -i '/^[ \\t]*\$/d' final_list
  end=\$(cat list_to_check)
  echo "" >> list_to_check
done

## Loop for check and download dependencies
rmdir --ignore-fail-on-non-empty /data/repo/pkg/_tdir
sed -i '/^[ \\t]*\$/d' final_list
sort -u final_list > temp
mv -f temp final_list
echo "" >> final_list
rm -f list_to_check
while read pkg; do
  if [[ "" == "\$pkg" ]]; then
    continue
  fi
  echo -e "\\033[36m =====> Check \$pkg... \\033[0m"
  ## Check if a package exists
  res=\$(apt-cache show \$pkg | grep "Package: \$pkg")
  if [[ "\$res" != "" ]]; then
    ## Download existed package
    echo -e "\\033[35m =====> Downloading package \$pkg... \\033[0m"
    res=\$(apt-get -R /data/repo/pkg/_tdir download \$pkg | grep 'E:' | sed -n 's/E://')
    if [[ "\$res" == "" ]]; then
      print_log "\$pkg : OK"
    else
      echo -e "\\033[31;1;4m \$res \\033[0m"
      print_log "\$pkg: \$res"
      if [[ "\$DEP_FILE" != "" ]]; then
        echo "=======================================" >> errors.log
        echo "\$pkg: \$res" >> errors.log
        grep ":\$pkg" \$DEP_FILE >> errors.log
      fi     
    fi
  else
    ## Put error message if package doesn't exist
    echo -e "\\033[31;1;4m\$pkg: No package found \\033[0m"
    print_log "\$pkg: No package found"
    if [[ "\$DEP_FILE" != "" ]]; then
      echo "=======================================" >> errors.log
      echo "\$pkg: No package found" >> errors.log
      grep ":\$pkg" \$DEP_FILE >> errors.log
    fi
  fi
done < final_list

## Downloading *.deb packages
echo -e "\\033[32;1m =====> Uploading *.deb packages... \\033[0m"
reprepro -b /data/repo --export=never -C main includedeb du2du4-prebuild /data/repo/pkg/*.deb

## Loop for download sources
while read component; do
  if [[ "" == "\$component" ]]; then
    continue
  fi
  echo -e "\\033[33;1m =====> Downloading sources for \$component... \\033[0m"
  apt-get source --download-only \$component
  echo -e "\\033[32;1m =====> Uploading sources for \$component... \\033[0m"
  reprepro -b /data/repo --export=never -C main includedsc du2du4-prebuild \$component*.dsc
  rm -rf \$component*
done < list_component

## Creating index for local repo
echo -e "\\033[32;1m =====> Creating indices for local repo... \\033[0m"
reprepro -Vb /data/repo export du2du4-prebuild
echo -e "\\033[32;1m =====> DONE \\033[0m"
if [[ "\$LOG_FILE" != "" ]]; then
  echo "Downloaded package list is put to \$LOG_FILE"
fi
EOT

cat > /etc/apt/sources.list <<EOT
deb http://10.20.50.188/dilos4 du4-unstable main contrib non-free
deb http://10.20.50.188/dilos dg2-unstable main contrib non-free
deb-src http://10.20.50.188/dilos4 du4-unstable main contrib non-free
EOT
apt-get update

If you are connected by SSH and would like to turn off SSH session or even temporary turn off your computer, then you have enter the following command for tests running:

screen ./make_repo.sh

Then you can in any time press Ctrl-A d and after that close your SSH session.

When you would like to get back, login again to SSH please and enter the command:

screen -r

cat > prepare.sh <<EOT
#!/bin/bash
WORKSPACE=/export/home/admin/du2du4

apt-get clean
apt-get autoclean
apt-get remove libiconv-dev -y

apt-get install -y acl iputils-ping kbd ksh scheduler system-data-console-fonts git
apt-get install -y traceroute gmake build-essential man-db po-debconf po4a debhelper
apt-get install -y libx11-6 libxi6 libxtst6 liblua5.2-dev libnghttp2-dev libjansson-dev
apt-get install -y mawk doxygen graphviz libdb-dev libgdbm-dev libsqlite3-dev libpq-dev
apt-get install -y bison dh-exec cmake docbook-xml docbook-xsl libgtest-dev screen

#/usr/bin/ln -fs libX11.so.6 /usr/lib/64/libX11.so.4
#/usr/bin/ln -fs libXi.so.6 /usr/lib/64/libXi.so.5
#/usr/bin/ln -fs libXtst.so.6 /usr/lib/64/libXtst.so.1
 bison dh-exec cmake docbook-xml docbook-xsl libgtest-dev screen
cp /etc/apt/sources.list /etc/apt/sources.list.bak
echo "" > /etc/apt/sources.list
echo "deb [trusted=yes] http://192.168.0.2/dilos du2du4-prebuild main contrib non-free" >> /etc/apt/sources.list
echo "deb-src [trusted=yes] http://192.168.0.2/dilos du2du4-prebuild main contrib non-free" 2>/dev/null >> /etc/apt/sources.list
cp /etc/apt/sources.list /etc/apt/sources.list.dev

apt-get update -y

#test -f /var/lib/dpkg/lock && rm -f /var/lib/dpkg/lock
#test -f /var/cache/apt/archives/lock && rm -f /var/cache/apt/archives/lock
#dpkg --configure -a
rm -rf \${WORKSPACE} && mkdir -p \${WORKSPACE}

apt-get upgrade -y

env DEBIAN_FRONTEND=noninteractive apt-get install -y -f
env DEBIAN_FRONTEND=noninteractive apt-get install -y git openssh-server
EOT

cat > build_all.sh <<EOT
#!/bin/bash

BLD_PARALLEL="4"
WORKSPACE=/export/home/admin/du2du4

#STOP_ON_ERROR=1
STOP_ON_ERROR=0
#GCC_COMPILER="gcc-610"
GCC_COMPILER=""

# Build a component: \$1 - component name, \$2 - stop-on-error flag
function build_component {
  BLD_RESULT="/data/repo/du/\$1"
  test -z "\$1" && exit 1
  test -d \${BLD_RESULT} && rm -rf \${BLD_RESULT}
  DEB_BUILD_OPTIONS="nocheck notest parallel=\${BLD_PARALLEL}"
  export DEB_BUILD_OPTIONS
  test -d \${WORKSPACE}/\$1 && rm -rf \${WORKSPACE}/\$1
  mkdir -p \${WORKSPACE}/\$1
  cd \${WORKSPACE}/\$1
  echo "====================== \$1 ======================"  | tee berror.log
  if [[ \$2 -eq 0 ]]; then
    apt-get build-dep -y \$1 >> berror.log || return 0 2>&1
    apt-get source --compile \$1 >> berror.log || return 0 2>&1
  else
    apt-get build-dep -y \$1 >> berror.log || return 1 2>&1
    apt-get source --compile \$1 >> berror.log || return 1 2>&1
  fi
  mkdir -p \${BLD_RESULT}
  cp -ax *.deb \${BLD_RESULT}/ || exit 1
  cp -ax *.changes \${BLD_RESULT}/ || exit 1
  cp -ax *.buildinfo \${BLD_RESULT}/ || exit 1

}

if [[ "\$GCC_COMPILER" != "" ]]; then
  echo "====================== Building GCC ======================"
  build_component \$GCC_COMPILER 1
  if [[ \$? -eq 1 ]]; then
    exit 1
  fi
  dpkg -i \$WORKSPACE/\$GCC_COMPILER/*.deb
fi

while read COMP; do
  if [[ "" == "\$COMP" ]] || [[ "\$GCC_COMPILER" == "\$COMP" ]]; then
    continue
  fi
  build_component \$COMP \$STOP_ON_ERROR
  if [[ \$? -eq 1 ]]; then
    exit 1
  fi
done < /data/repo/pkg/list_component
EOT

If you are connected by SSH and would like to turn off SSH session or even temporary turn off your computer, then you have enter the following command for tests running:

screen ./build_all.sh

Then you can in any time press Ctrl-A d and after that close your SSH session.

When you would like to get back, login again to SSH please and enter the command:

screen -r

cat > check_fails.sh <<EOT
#!/bin/bash

#WDIR=\$1
WDIR=/export/home/du4

subdirs=\$(ls -1 \$WDIR)

for dr in \$subdirs
do
  res=\$(tail -1 \$WDIR/\$dr/berror.log)
  if [[ "\$(echo \$res | grep 'Fetched')" == "" ]]; then
    echo "\$dr : \$res"
  fi
done
EOT
chmod 777 ./check_fails.sh
./check_fails.sh

Note : If you need first to build the GCC compiler and use it to build other components, then put the compiler component name to the GCC_COMPILER variable, for example:

GCC_COMPILER="gcc-610". Its exact name you can find in the list_component file.