Pack demos.sh

(Różnice między wersjami)
Skocz do: nawigacji, wyszukiwania
(Info)
(Kod: nowa wersja)
Linia 29: Linia 29:
 
<bash>
 
<bash>
 
#!/bin/bash
 
#!/bin/bash
 
+
#
 
# _KaszpiR_ kaszpir@gmail.com
 
# _KaszpiR_ kaszpir@gmail.com
 
# Script to find and pack HLTV demos
 
# Script to find and pack HLTV demos
 
# if you want to change it to something else you must edit awk lines where the year/month/day is extracted from filename
 
# if you want to change it to something else you must edit awk lines where the year/month/day is extracted from filename
 
+
#
 +
# run program with any parameter to see the list of  files that will be processed
 +
#
 +
# Changelog:
 +
# ver 1.3 2011.01.20, 14:16
 +
# + added basic checks to avoid error configurations, should be really more
 +
# + running script with any parameter will show command and will require pressing any key to continue
 +
# + added quotation marks in some places to allow specific name patterns
 +
# ver 1.2 2011.01.16, 15:15
 +
# + added ftp upload
 +
# + all variables curly brackets for security reasons
 +
# + listing old files is potional
 +
# + added hashes on output lines for easier parsing
 
# ver 1.1 2009.08.31
 
# ver 1.1 2009.08.31
 
# + added option to list directories for deletion older than X days
 
# + added option to list directories for deletion older than X days
 
# + added definiton for the extension files
 
# + added definiton for the extension files
 +
##################################################################
 +
 +
WORKDIR="/home/kaszpir/eggdrop/"; # enter this directory when executing any commands and creating TMPFILE
 +
 +
SRCDIR="/home/kaszpir/eggdrop/hlds/logs/"; # source directory where to find files for packing
 +
DSTDIR="/home/kaszpir/out/"; # destination directory where to put packed files
 +
NAMEPATTERN="hlds.pl*"; # file names to search when finding files
 +
EXTENSION=""; # search only for files with this extension, should be the same as used in NAMEPATTERN
  
WORKDIR="/home/hlds/scripts"
+
# for bzip2 packing try those
 +
PACKER=`which tar`; # what packing program to use
 +
PACK_OPT="-Pjcf"; # packing program options
 +
PACK_EXT="tar.bz2"; # extension to use when creating archives
  
SRCDIR="/home/hlds/hlds/dmc"
+
# example for 7zip
DSTDIR="/home/hlds/scripts/demos"
+
#PACKER=`which /usr/bin/7z`; # what packing program to use
NAMEPATTERN="hltv*.dem"
+
#PACK_OPT="a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on"; # packing program options
EXTENSION=".dem"
+
#PACK_EXT="7z"; # extension to use when creating archives
  
PACKER=`which tar`
+
TMPFILE=".pack_logs_hlds.tmp"; # name of the temporary file to store names of the processed files
PACK_OPT="-Pjcf"
+
PACK_EXT="tar.bz2"
+
TMPFILE=".pack.tmp"
+
LOGFILE="pack.log"
+
NICE=20        ;# the niceness level, 20 = lowest, do it when cpu is idle , 0 = highest, do it as fast as possible by current user
+
DELETE_OLD=1    ;#list dirs marked for deletion
+
OLDER_THAN=14  ; # 14 days
+
  
date >> $LOGFILE
 
echo "================"
 
cd $WORKDIR
 
  
rm -f $TMPFILE
+
LOGFILE="pack_logs_hlds.log"; # file to log actions
 +
SILENT=1; # do not show some messages (to decrease the produced crap in crontab results via mail
 +
NICE=20 ; # the niceness level, 20 = lowest, do it when cpu is idle , 0 = highest, do it as fast as possible by current user
 +
LIST_OLD=1; #list old files, useful for scripts
 +
LIST_OLD_FILE="pack_logs_hlds_old.txt"; # destination where to print old file list
 +
DELETE_OLD=0    ; # defines if the script should delete old packed files
 +
OLDER_THAN=7  ;  # 14 days
  
 +
FTP_USE=1; # use ftp to upload lohs somwhere after packing them
 +
FTP_DEBUG=0; # generate ftp transfer log with debug messages
 +
FTP_LOG="ftp_211.log";
 +
 +
FTP_CREDENTIALS="${WORKDIR}/.secret"; # ncftp credentials file in format, refer to ncftpput -f parameter, read the ncftp manual, make sure this file is chmod 600
 +
#
 +
# format of the FTP_CREDENTIALS file (plaintext)
 +
# host some.host.com
 +
# user lamer
 +
# pass dumbpassword
 +
#
 +
 +
FTP_PATH="/"; # remore directory where to store logs
 +
FTP_CLIENT=`which ncftpput`; # program to run ncftpput (part of ncftp packagae)
 +
FTP_TRIES=10;
 +
FTP_OPTS="-S .tmp";# extra ftp upload options, refer to manual
 +
 +
 +
##################################################################
 +
# Start of sript
 +
LC_ALL=C
 +
date >> ${LOGFILE}
 +
echo "# ================"
 +
 +
# sanity checks , very basic ones
 +
if [ ! -d ${WORKDIR} ]; then
 +
echo "ERROR: WORKDIR: ${WORKDIR} is not a directory.";
 +
exit 1;
 +
fi
 +
if [ ! -d ${SRCDIR} ]; then
 +
echo "ERROR: SRCDIR: ${SRCDIR} is not a directory.";
 +
exit 1;
 +
fi
 +
if [ ! -d ${DSTDIR} ]; then
 +
echo "ERROR: DSTDIR: ${DSTDIR} is not a directory.";
 +
exit 1;
 +
fi
 +
if [ ! -w ${DSTDIR} ]; then
 +
echo "ERROR: DSTDIR: ${DSTDIR} is not writable.";
 +
exit 1;
 +
fi
 +
if [ "${PACKER}" == "" ]; then
 +
echo "ERROR: PACKER: is not found, check if you got apropiate packer program installed."
 +
exit 1;
 +
fi
 +
if [ "${FTP_USE}" == "1" ]; then
 +
if [ ! -r ${FTP_CREDENTIALS} ]; then
 +
echo "ERROR: FTP_CREDENTIALS: ${FTP_CREDENTIALS} is not found, or is not readable."
 +
exit 1;
 +
fi
 +
if [ "${FTP_CLIEN}" == "" ]; then
 +
echo "ERROR: FTP_CLIEN: is not found, check if you got apropiate packer program installed."
 +
exit 1;
 +
fi
 +
 +
fi
 +
 +
cd ${WORKDIR}
 +
 +
rm -f ${TMPFILE}
 +
 +
##################################################################
 
#this is just for checking what the script will do , or debug mode
 
#this is just for checking what the script will do , or debug mode
 
if [ $1 ]; then
 
if [ $1 ]; then
find $SRCDIR -maxdepth 1 -type f -name $NAMEPATTERN -mmin +60
+
echo "find ${SRCDIR} -maxdepth 1 -type f -name \"${NAMEPATTERN}\" -mmin +120"
exit
+
pause
 +
find ${SRCDIR} -maxdepth 1 -type f -name "${NAMEPATTERN}" -mmin +120
 +
 
 +
exit 0
 
fi
 
fi
 +
##################################################################
 +
# real run
  
 
#find all files modifed over hour ago, so we gonna ignore the currently recorded demo
 
#find all files modifed over hour ago, so we gonna ignore the currently recorded demo
find $SRCDIR -maxdepth 1 -type f -name $NAMEPATTERN -mmin +60 > $TMPFILE
+
find ${SRCDIR} -maxdepth 1 -type f -name "${NAMEPATTERN}" -mmin +120 > ${TMPFILE}
  
 
OYEAR=""
 
OYEAR=""
 
OMONTH=""
 
OMONTH=""
 
ODAY=""
 
ODAY=""
TOTAL=`wc -l $TMPFILE|awk '{print $1}'`
+
TOTAL=`wc -l ${TMPFILE}|awk '{print $1}'`
echo "$TOTAL files."
+
echo "# ${TOTAL} files."
 
INCR=0;
 
INCR=0;
for f in `cat $TMPFILE` ; do
+
for f in `cat ${TMPFILE}` ; do
 
     # display enumerator on screen, for better view if the progress is made
 
     # display enumerator on screen, for better view if the progress is made
 
     INCR=$((INCR + 1))
 
     INCR=$((INCR + 1))
     echo "$INCR / $TOTAL"
+
     if [ ${SILENT} -lt 1 ]; then
 +
echo "# ${INCR} / ${TOTAL}"
 +
    fi
 
     # split filename to daytime format, and used to produce subdirectories and pack files into the proper dir
 
     # split filename to daytime format, and used to produce subdirectories and pack files into the proper dir
     YEAR=`echo $f | awk -F- '{print substr($2,1,2)}'`
+
     YEAR=`echo ${f} | awk -F- '{print substr($2,1,4)}'`
     MONTH=`echo $f | awk -F- '{print substr($2,3,2)}'`
+
     MONTH=`echo ${f} | awk -F- '{print substr($2,5,2)}'`
     DAY=`echo $f | awk -F- '{print substr($2,5,2)}'`
+
     DAY=`echo ${f} | awk -F- '{print substr($2,7,2)}'`
     if [ "$DAY" != "$ODAY" ];then
+
     if [ "${DAY}" != "${ODAY}" ];then
 
       # detect the day change and make new subdirectory
 
       # detect the day change and make new subdirectory
       FULLDATE="20$YEAR/$MONTH/$DAY"
+
       FULLDATE="${YEAR}/${MONTH}/${DAY}"
       mkdir -p $DSTDIR/$FULLDATE
+
       mkdir -p ${DSTDIR}/${FULLDATE}
       ODAY=$DAY
+
       ODAY=${DAY}
 
     fi
 
     fi
     SUBNAME=`basename $f $EXTENSION`
+
     SUBNAME=`basename ${f} ${EXTENSION}`
 
     # run with lowest priority to avoid cpu usage
 
     # run with lowest priority to avoid cpu usage
     nice -n$NICE $PACKER $PACK_OPT $DSTDIR/$FULLDATE/$SUBNAME.$PACK_EXT $f >> $LOGFILE 2>&1
+
     nice -n${NICE} ${PACKER} ${PACK_OPT} ${DSTDIR}/${FULLDATE}/${SUBNAME}.${PACK_EXT} ${f} >> ${LOGFILE} 2>&1
 
+
 
     if [ $? -eq 0 ]; then
 
     if [ $? -eq 0 ]; then
 
         # packing completed, delete the source file
 
         # packing completed, delete the source file
         rm -f $f
+
         rm -f ${f}
 
     else
 
     else
 
         #packing failed, we leave the source file as it was
 
         #packing failed, we leave the source file as it was
         echo "ERROR: not deleting $f"
+
         echo "# WARNING: not deleting due to compression error: ${f}"
 
     fi
 
     fi
 
+
 
done
 
done
  
if [ $DELETE_OLD == "1" ]; then
+
##################################################################
   echo "Listing old files"
+
#Listing old files
   #           delete days! del dirs  full day  older than                    print it just in case
+
if [ ${LIST_OLD} == "1" ]; then
   # mindepth is REQUIRED otherwise you will delete wrong directories, like whole current year or month
+
   echo "# Listing old files:"
   find $DSTDIR -mindepth 3 -type d -daystart -mtime +$OLDER_THAN -print0 | xargs -0 echo
+
  find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} > ${LIST_OLD_FILE}
 +
 
 +
   #find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN}
 +
# the above line was not tested
 +
fi
 +
##################################################################
 +
# Deleting old files
 +
if [ ${DELETE_OLD} == "1" ]; then
 +
   echo "# Deleting old files:"
 +
#echo "find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} -print0 | xargs -0 rm -rf"
 +
   find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} -print0 | xargs -0 rm -rf
 
fi
 
fi
  
echo "================"
+
##################################################################
 +
# ftp without removing old files
 +
# -R recursive
 +
# -z try to resume broken uploads
 +
if [ ${FTP_USE} == "1" ]; then
 +
echo "# FTP process starts:"
 +
if [ ${FTP_DEBUG} == "1" ]; then
 +
# -d file , generate debug log
 +
${FTP_CLIENT} ${FTP_OPTS} -d ${FTP_LOG} -f ${FTP_CREDENTIALS} -r ${FTP_TRIES} -z -R ${FTP_PATH} ${DSTDIR}/
  
 +
else
 +
# no log
 +
# -DD delete files after successful uploading
 +
${FTP_CLIENT} ${FTP_OPTS} -f ${FTP_CREDENTIALS} -r ${FTP_TRIES} -z -DD -R ${FTP_PATH} ${DSTDIR}/
 +
fi
 +
echo "# FTP process ends."
 +
fi
 +
 +
 +
 +
# removing old files
 +
#
 +
# get recursively directories , check date and rape
 +
# execute rm  based on local directory structure
 +
#
 +
#
 +
##################################################################
 +
# End of script
 +
echo "# ================="
 +
##################################################################
  
 
</bash>
 
</bash>
 +
 
=Info=
 
=Info=
 
No dobra, jak ktoś przeczytał kod to się pewnie kapnął że nie będzie kasowanych katalogów ze starymi spakownaymi demami, bo jest komenda echo a nie rm -rf
 
No dobra, jak ktoś przeczytał kod to się pewnie kapnął że nie będzie kasowanych katalogów ze starymi spakownaymi demami, bo jest komenda echo a nie rm -rf
  
 
Tyle, że trzeba BARDZO uważać co się będzie kasować, dlatego dałem echo.
 
Tyle, że trzeba BARDZO uważać co się będzie kasować, dlatego dałem echo.

Wersja z 15:54, 20 sty 2011

Opis

Potrzebowałem pakować dema do Deathmatch Classic. Zapomniałem go tu wrzucić.

  • daje możliwość tez automatycznego pakowania i segregowania dem wg daty rok/miesiac/dzien
  • nie pakuje aktualnie nagrywanych dem
  • nie usuwa źle spakowanego dema (np jak sobie dysk zapchasz :) )
  • może usuwać stare spakowane dema, np starsze niż 14 dni

Parametry w konfiguracji, linia po linii (bez komentarzy):

  • konfiguracja ścieżki w której znajduje się skrypt (WORKDIR)
  • konfiguracja gdzie znajdują się dema do spakowania (SRCDIR)
  • ścieżka gdzie zapisywać dema w formacie (DSTDIR)/rok/miesiac/dzien/
  • wzorzec rozpoznawania dem, domyślnie hltv*.dem
  • wzorzec rozszerzenia nazwy pliku dem - tego lepiej nie ruszaj
  • wybór pakera, domyślnie tar
  • opcje pakera, domyślnie dla bzip2
  • rozszerzenie pakowanego dema, domyślnie tar.bz2
  • plik tymczasowy do listy plików do spakowania
  • log z wykonanych operacji
  • ustawienie priorytetu aby nie zamulać procka (NICE)
  • opcja kasacji starych spakowanych dem DELETE_OLD
  • spakowane dema starsze niż X dni są kasowane (domyślnie 14)

Do crona wpisać cos w ten deseń:

20 5 * * * /home/hlds/scripts/pack_demos.sh > /home/hlds/scripts/pack_demos.cron.log 2>&1

Jak włączysz mózg i poczytasz manuala to dostosujesz do dowolnego packera i pakowania dowolnie pojawiających się cyklicznie plików (chociażby log, aczkolwiek tutaj lepiej byłoby pakować logi dniami a nie każde osobno)

Kod

#!/bin/bash
#
# _KaszpiR_ kaszpir@gmail.com
# Script to find and pack HLTV demos
# if you want to change it to something else you must edit awk lines where the year/month/day is extracted from filename
#
# run program with any parameter to see the list of  files that will be processed
#
# Changelog:
# ver 1.3 2011.01.20, 14:16 
# + added basic checks to avoid error configurations, should be really more
# + running script with any parameter will show command and will require pressing any key to continue
# + added quotation marks in some places to allow specific name patterns
# ver 1.2 2011.01.16, 15:15 
# + added ftp upload
# + all variables curly brackets for security reasons
# + listing old files is potional
# + added hashes on output lines for easier parsing
# ver 1.1 2009.08.31
# + added option to list directories for deletion older than X days
# + added definiton for the extension files
##################################################################
 
WORKDIR="/home/kaszpir/eggdrop/"; # enter this directory when executing any commands and creating TMPFILE
 
SRCDIR="/home/kaszpir/eggdrop/hlds/logs/"; # source directory where to find files for packing
DSTDIR="/home/kaszpir/out/"; # destination directory where to put packed files
NAMEPATTERN="hlds.pl*"; # file names to search when finding files
EXTENSION=""; # search only for files with this extension, should be the same as used in NAMEPATTERN
 
# for bzip2 packing try those
PACKER=`which tar`; # what packing program to use 
PACK_OPT="-Pjcf"; # packing program options
PACK_EXT="tar.bz2"; # extension to use when creating archives
 
# example for 7zip
#PACKER=`which /usr/bin/7z`; # what packing program to use 
#PACK_OPT="a -t7z -m0=lzma -mx=9 -mfb=64 -md=32m -ms=on"; # packing program options
#PACK_EXT="7z"; # extension to use when creating archives
 
TMPFILE=".pack_logs_hlds.tmp"; # name of the temporary file to store names of the processed files
 
 
LOGFILE="pack_logs_hlds.log"; # file to log actions
SILENT=1; # do not show some messages (to decrease the produced crap in crontab results via mail
NICE=20 ; # the niceness level, 20 = lowest, do it when cpu is idle , 0 = highest, do it as fast as possible by current user
LIST_OLD=1; #list old files, useful for scripts
LIST_OLD_FILE="pack_logs_hlds_old.txt"; # destination where to print old file list
DELETE_OLD=0    ; # defines if the script should delete old packed files
OLDER_THAN=7   ;  # 14 days
 
FTP_USE=1; # use ftp to upload lohs somwhere after packing them
FTP_DEBUG=0; # generate ftp transfer log with debug messages
FTP_LOG="ftp_211.log";
 
FTP_CREDENTIALS="${WORKDIR}/.secret"; # ncftp credentials file in format, refer to ncftpput -f parameter, read the ncftp manual, make sure this file is chmod 600
#
# format of the FTP_CREDENTIALS file (plaintext)
# host some.host.com
# user lamer
# pass dumbpassword
#
 
FTP_PATH="/"; # remore directory where to store logs
FTP_CLIENT=`which ncftpput`; # program to run ncftpput (part of ncftp packagae)
FTP_TRIES=10;
FTP_OPTS="-S .tmp";# extra ftp upload options, refer to manual
 
 
##################################################################
# Start of sript
LC_ALL=C
date >> ${LOGFILE}
echo "# ================"
 
# sanity checks , very basic ones
if [ ! -d ${WORKDIR} ]; then
	echo "ERROR: WORKDIR: ${WORKDIR} is not a directory.";
	exit 1;
fi
if [ ! -d ${SRCDIR} ]; then
	echo "ERROR: SRCDIR: ${SRCDIR} is not a directory.";
	exit 1;
fi
if [ ! -d ${DSTDIR} ]; then
	echo "ERROR: DSTDIR: ${DSTDIR} is not a directory.";
	exit 1;
fi
if [ ! -w ${DSTDIR} ]; then
	echo "ERROR: DSTDIR: ${DSTDIR} is not writable.";
	exit 1;
fi
if [ "${PACKER}" == "" ]; then
	echo "ERROR: PACKER: is not found, check if you got apropiate packer program installed."
	exit 1;
fi
if [ "${FTP_USE}" == "1" ]; then
	if [ ! -r ${FTP_CREDENTIALS} ]; then
		echo "ERROR: FTP_CREDENTIALS: ${FTP_CREDENTIALS} is not found, or is not readable."
		exit 1;
	fi	
	if [ "${FTP_CLIEN}" == "" ]; then
		echo "ERROR: FTP_CLIEN: is not found, check if you got apropiate packer program installed."
		exit 1;
	fi
 
fi
 
cd ${WORKDIR}
 
rm -f ${TMPFILE}
 
##################################################################
#this is just for checking what the script will do , or debug mode
if [ $1 ]; then
echo "find ${SRCDIR} -maxdepth 1 -type f -name \"${NAMEPATTERN}\" -mmin +120"
pause 
find ${SRCDIR} -maxdepth 1 -type f -name "${NAMEPATTERN}" -mmin +120 
 
exit 0
fi
##################################################################
# real run
 
#find all files modifed over hour ago, so we gonna ignore the currently recorded demo
find ${SRCDIR} -maxdepth 1 -type f -name "${NAMEPATTERN}" -mmin +120 > ${TMPFILE}
 
OYEAR=""
OMONTH=""
ODAY=""
TOTAL=`wc -l ${TMPFILE}|awk '{print $1}'`
echo "# ${TOTAL} files."
INCR=0;
for f in `cat ${TMPFILE}` ; do
    # display enumerator on screen, for better view if the progress is made
    INCR=$((INCR + 1))
    if [ ${SILENT} -lt 1 ]; then
			echo "# ${INCR} / ${TOTAL}"
    fi
    # split filename to daytime format, and used to produce subdirectories and pack files into the proper dir
    YEAR=`echo ${f} | awk -F- '{print substr($2,1,4)}'`
    MONTH=`echo ${f} | awk -F- '{print substr($2,5,2)}'`
    DAY=`echo ${f} | awk -F- '{print substr($2,7,2)}'`
    if [ "${DAY}" != "${ODAY}" ];then
      # detect the day change and make new subdirectory
      FULLDATE="${YEAR}/${MONTH}/${DAY}"
      mkdir -p ${DSTDIR}/${FULLDATE}
      ODAY=${DAY}
    fi
    SUBNAME=`basename ${f} ${EXTENSION}`
    # run with lowest priority to avoid cpu usage
    nice -n${NICE} ${PACKER} ${PACK_OPT} ${DSTDIR}/${FULLDATE}/${SUBNAME}.${PACK_EXT} ${f} >> ${LOGFILE} 2>&1
 
    if [ $? -eq 0 ]; then
        # packing completed, delete the source file
        rm -f ${f}
    else
        #packing failed, we leave the source file as it was
        echo "# WARNING: not deleting due to compression error: ${f}"
    fi
 
done
 
##################################################################
#Listing old files
if [ ${LIST_OLD} == "1" ]; then
  echo "# Listing old files:"
  find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} > ${LIST_OLD_FILE} 
 
  #find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} 
	# the above line was not tested
fi
##################################################################
# Deleting old files
if [ ${DELETE_OLD} == "1" ]; then
  echo "# Deleting old files:"
 #echo "find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} -print0 | xargs -0 rm -rf"
  find ${DSTDIR} -mindepth 3 -type d -daystart -mtime +${OLDER_THAN} -print0 | xargs -0 rm -rf
fi
 
##################################################################
# ftp without removing old files
# -R recursive
# -z try to resume broken uploads
if [ ${FTP_USE} == "1" ]; then
echo "# FTP process starts:"
	if [ ${FTP_DEBUG} == "1" ]; then
	# -d file , generate debug log
	${FTP_CLIENT} ${FTP_OPTS} -d ${FTP_LOG} -f ${FTP_CREDENTIALS} -r ${FTP_TRIES} -z -R ${FTP_PATH} ${DSTDIR}/
 
	else
	# no log
	# -DD delete files after successful uploading
	${FTP_CLIENT} ${FTP_OPTS} -f ${FTP_CREDENTIALS} -r ${FTP_TRIES} -z -DD -R ${FTP_PATH} ${DSTDIR}/
	fi
echo "# FTP process ends."
fi
 
 
 
# removing old files
# 
# get recursively directories , check date and rape
# execute rm  based on local directory structure 
#
#
##################################################################
# End of script 
echo "# ================="
################################################################## 

Info

No dobra, jak ktoś przeczytał kod to się pewnie kapnął że nie będzie kasowanych katalogów ze starymi spakownaymi demami, bo jest komenda echo a nie rm -rf

Tyle, że trzeba BARDZO uważać co się będzie kasować, dlatego dałem echo.

Osobiste
Przestrzenie nazw
Warianty
Działania
HLDS.pl - Menu:
Inne
IRC
Inne sekcje:
Znajomi:
Narzędzia