commons_functions.ksh 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. ########################################################################
  2. ###
  3. ##
  4. #
  5. # Author: Stéphane HUC
  6. # mail: devs@stephane-huc.net
  7. # gpg:fingerprint: CE2C CF7C AB68 0329 0D20 5F49 6135 D440 4D44 BD58
  8. #
  9. # License: BSD Simplified 2 Clauses
  10. #
  11. # Github: https://framagit.org/BlackLists/BlockZones.git
  12. #
  13. # Date: 2017/07/16
  14. #
  15. ##
  16. ###
  17. ########################################################################
  18. ###
  19. ##
  20. # For *BSD: ksh, ksh93...
  21. ##
  22. ###
  23. ########################################################################
  24. ### Ty to @kuniyoshi for FreeBSD Codes - 2017/07/10
  25. ########################################################################
  26. ### Get data into file in array...
  27. build_blocklists() {
  28. display_mssg "#" "*** Manage list: ${DIR_SRC}/${list} ***"
  29. if [ -f "${DIR_SRC}/${list}" ]; then
  30. i=0
  31. while read -r line; do
  32. if echo "${line}" | grep -v "^#"; then blocklists[$i]="${line}"; fi
  33. let i++
  34. done < "${DIR_SRC}/${list}"
  35. unset i
  36. else
  37. display_mssg "KO" "The file ${DIR_SRC}/${list} seems not to be readable!"
  38. byebye
  39. fi
  40. }
  41. ### Create sha512 checksums files!
  42. build_sums() {
  43. if [ "${one_checksum_file}" = 0 ]; then
  44. typeset bool=1
  45. case "${list}" in
  46. "bogons") output="${file}" ;;
  47. esac
  48. if [ -f "${DIR_LISTS}/${output}" ]; then
  49. cd "${DIR_LISTS}" || exit 1
  50. case "${OSN}" in
  51. "FreeBSD")
  52. if sha512 "${output}" > "${output}.sha512"; then bool=0; fi
  53. ;;
  54. "OpenBSD")
  55. if sha512 -h "${output}.sha512" "${output}"; then bool=0; fi
  56. ;;
  57. esac
  58. if [ ${bool} -eq 0 ]; then
  59. display_mssg "OK" "The checksum file '${DIR_LISTS}/${output}' is correctly created!"
  60. create_sign
  61. else
  62. display_mssg "KO" "It seems to have a problem to create checksum file ${DIR_LISTS}/${output}!"
  63. fi
  64. cd "${ROOT}" || exit 1
  65. fi
  66. unset bool
  67. fi
  68. }
  69. byebye() {
  70. display_mssg "KO" " /!\ Script stop here /!\ "
  71. display_mssg "KO" "Please, search to understand reasons."
  72. #export $OLD_TERM
  73. exit 1;
  74. }
  75. check_needed_softs() {
  76. # curl
  77. if [ -f /usr/local/bin/curl ]; then
  78. use_curl=1
  79. else
  80. display_mssg "hg" "For the next time, if you installed the curl tool, the script will attempt to use-it!"
  81. sleep 1
  82. fi
  83. # wget
  84. if [ -f /usr/local/bin/wget ]; then
  85. use_wget=1
  86. else
  87. display_mssg "hg" "For the next time, if you installed the wget tool, the script will attempt to use-it... only if curl tool is not installed!"
  88. sleep 1
  89. fi
  90. # signify
  91. if [ "${OSN}" = "FreeBSD" ]; then
  92. if [ ! -f /usr/bin/signify ]; then
  93. use_sign=0
  94. display_mssg "KO" " /!\ It seems signify tool is not available /!\ "
  95. if confirm "Do you want to continue without sign lists?"; then
  96. display_mssg "#" "The $0 script continue without generate sign file!"
  97. sleep 1
  98. else
  99. display_mssg "#" "Please, install signify tool!"
  100. byebye
  101. fi
  102. fi
  103. fi
  104. }
  105. confirm () {
  106. read -r response?"${1} [y|n] "
  107. case "$response" in
  108. y|Y|o|O|1) # O is not zero: O(ui) ;)
  109. true
  110. ;;
  111. *)
  112. false
  113. ;;
  114. esac
  115. }
  116. create_one_sums() {
  117. if [ "${one_checksum_file}" = 1 ]; then
  118. cd "${DIR_LISTS}" || exit 1
  119. find ./ -exec sha512 {} + > "${DIR_SRC}/BlockZones.sha512";
  120. if [ -f "${DIR_SRC}/BlockZones.sha512" ]; then
  121. display_mssg "OK" "The checksum file '${DIR_SRC}/BlockZones.sha512' is correctly created!"
  122. if mv "${DIR_SRC}/BlockZones.sha512" "${DIR_LISTS}"; then
  123. display_mssg "OK" "The checksum file '${DIR_SRC}/BlockZones.sha512' is correctly moved to ${DIR_LISTS}!"
  124. create_sign
  125. else
  126. display_mssg "KO" "Error to move checksum file '${DIR_SRC}/BlockZones.sha512' into ${DIR_LISTS}!"
  127. fi
  128. else
  129. display_mssg "KO" "It seems to have a problem to create checksum file '${DIR_SRC}/BlockZones.sha512'!"
  130. fi
  131. cd "${ROOT}" || exit 1
  132. fi
  133. }
  134. create_sign() {
  135. if [ "${use_sign}" = 1 ]; then
  136. if [ "${one_checksum_file}" = 1 ]; then
  137. if signify -S -s "${dir_sec_key_signify}" -m "${DIR_LISTS}/BlockZones.sha512" -e -x "${DIR_LISTS}/BlockZones.sha512.sig"; then
  138. display_mssg "OK" "The sign file '${DIR_LISTS}/BlockZones.sha512.sig' is correctly created!"
  139. else
  140. display_mssg "KO" "It seems to have a problem to create sign file '${DIR_LISTS}/BlockZones.sha512.sign'!"
  141. fi
  142. else
  143. if signify -S -s "${dir_sec_key_signify}" -m "${DIR_LISTS}/${output}.sha512" -e -x "${DIR_LISTS}/${output}.sha512.sig"; then
  144. display_mssg "OK" "The sign file '${DIR_LISTS}/${output}.sha512.sig' is correctly created!"
  145. else
  146. display_mssg "KO" "It seems to have a problem to create sign file '${DIR_LISTS}/${output}.sha512.sign'!"
  147. fi
  148. fi
  149. fi
  150. }
  151. ### To delete files
  152. del_files() {
  153. display_mssg "#" "########## Deleting old files ##########"
  154. for f in "${FILES[@]}"; do
  155. echo "file: $f"
  156. if [ -f "${f}" ]; then
  157. if rm "${f}"; then
  158. display_mssg "OK" "The file ${f} is correctly deleted!"
  159. else
  160. display_mssg "KO" "It seems to have a problem to delete file: ${f}!"
  161. fi
  162. fi
  163. done
  164. del_sums
  165. }
  166. ### To delete spaces, lines into files
  167. del_spaces() {
  168. display_mssg "#" "########## Deleting empties spaces, lines... ##########"
  169. # delete empties spaces and lines...
  170. for f in "${FILES[@]}"; do
  171. { rm "${f}" && sed -e "/^$/d;/^[[:space:]]*$/d" | sort -du -o "${f}"; } < "${f}"
  172. done
  173. }
  174. del_sums() {
  175. if [ "${one_checksum_file}" = 1 ]; then
  176. find "${DIR_LISTS}/" -type f -name "*\.sha512*" -exec rm -f {} +
  177. else
  178. rm -f "${DIR_LISTS}/BlockZones.sha512*"
  179. fi
  180. }
  181. display_mssg() {
  182. typeset statut info
  183. statut="$1" info="$2"
  184. case "${statut}" in
  185. "KO"|1) color="${red}" ;;
  186. "OK"|0) color="${green}" ;;
  187. esac
  188. if [ "${statut}" == "#" ]; then
  189. printf "[ ${bold}%s${neutral} ] \n" "${info}"
  190. elif [ "${statut}" == "hb" ]; then
  191. printf "${dim}%s${neutral} \n" "${info}"
  192. else
  193. printf "[ ${color}%s${neutral} ] %s \n" "${statut}" "${info}"
  194. fi
  195. unset info statut text
  196. }
  197. ### Downloader needed files
  198. download() {
  199. display_mssg "#" "########## Attempt to get blocklists files ##########"
  200. display_mssg "hb" "=> Attempt to download file: ${filename}"
  201. typeset bool=0
  202. if [ "${use_curl}" = 1 ]; then
  203. if ! /usr/local/bin/curl -A "Mozilla/5.0 (X11; Debian; Linux; rv:0.0) Gecko/20100101 Firefox/0.0" -o "${filename}" "${url}"; then
  204. bool=1
  205. fi
  206. elif [ "$(use wget)" = 1 ]; then
  207. if ! /usr/local/bin/wget --user-agent="Mozilla/5.0 (X11; Debian; Linux; rv:0.0) Gecko/20100101 Firefox/0.0" -c -O "${filename}" "${url}"; then
  208. bool=1
  209. fi
  210. else
  211. if ! /usr/bin/ftp -n -m -C -o "${filename}" "${url}"; then bool=1; fi
  212. fi
  213. if [ ${bool} -eq 0 ]; then
  214. display_mssg "OK" "The file ${filename} is correctly downloaded!"
  215. else
  216. display_mssg "KO" "It seems to have a problem with download file ${filename}!"
  217. byebye
  218. fi
  219. unset bool
  220. }
  221. make_uniq_list() {
  222. touch_files
  223. display_mssg "hb" "====> Attempt to make uniq file with filename: ${filename}"
  224. case "${list}" in
  225. "badips")
  226. display_mssg "hb" "=> Check ipv4!"
  227. valid_ipv4 "${filename}" >> "${DIR_SRC}/${list}_ipv4"
  228. display_mssg "hb" "=> Check ipv6!"
  229. valid_ipv6 "${filename}" >> "${DIR_SRC}/${list}_ipv6"
  230. ;;
  231. "bogons")
  232. case "${file}" in
  233. "fullbogons-ipv4.txt")
  234. display_mssg "hb" "=> Check ipv4!"
  235. valid_ipv4 "${filename}" >> "${DIR_LISTS}/${file}"
  236. ;;
  237. "fullbogons-ipv6.txt")
  238. display_mssg "hb" "=> Check ipv6!"
  239. valid_ipv6 "${filename}" >> "${DIR_LISTS}/${file}"
  240. ;;
  241. esac
  242. ;;
  243. "domains")
  244. awk '{ print tolower($0) }' "${filename}" >> "${DIR_SRC}/uniq_${list}"
  245. ;;
  246. esac
  247. del_spaces
  248. }
  249. ### To created files
  250. touch_files() {
  251. display_mssg "#" "########## Touch files ##########"
  252. for f in "${FILES[@]}"; do
  253. if [ ! -f "${f}" ]; then
  254. if touch "${f}"; then
  255. display_mssg "OK" "The file ${f} is correctly created!"
  256. else
  257. display_mssg "KO" "It seems to have a problem to created file: ${f}!"
  258. fi
  259. fi
  260. done
  261. }
  262. ### extract archive
  263. uncompress() {
  264. if [ "$(file -b -i "${filename}")" = "application/gzip" ]; then
  265. display_mssg "hb" "==> Attempt to extract archive .gz: ${filename}"
  266. gunzip -d -f -q "${filename}";
  267. fi
  268. if [ "$(file -b -i "${filename}")" = "application/x-gzip" ]; then
  269. display_mssg "hb" "==> Attempt to extract archive .gz: ${filename}"
  270. gunzip -d -f -q "${filename}";
  271. fi
  272. if [ "$(file -b -i "${filename}")" = "application/zip" ]; then
  273. display_mssg "hb" "==> Attempt to extract archive .zip: ${filename}"
  274. unzip -oqu "${filename}" -d "${filename%.zip}"
  275. fi
  276. }
  277. # functions valid_ip**() inspired
  278. # by: https://helloacm.com/how-to-valid-ipv6-addresses-using-bash-and-regex/
  279. valid_ipv4() {
  280. egrep '^([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\/?([0-9]{1,2})?)$' "${1}"
  281. }
  282. valid_ipv6() {
  283. egrep '^([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])$' "${1}"
  284. }