blacklists 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505
  1. #!/bin/bash
  2. #set -x
  3. [ -n "$TERM" ] && clear
  4. ###
  5. #
  6. # Author: Stéphane HUC
  7. # mail: devs@stephane-huc.net
  8. # gpg:fingerprint: CE2C CF7C AB68 0329 0D20 5F49 6135 D440 4D44 BD58
  9. #
  10. # License: GNU/LGPL
  11. #
  12. # Github: https://framagit.org/BlackLists/BlockZones.git
  13. #
  14. # Date: 2017/03/26
  15. #
  16. ###
  17. export LC_ALL=C
  18. IPv4=1 # enable or not IPV4 management - use by hosts, unbound
  19. IPv6=1 # enable or not IPv6 management - use by hosts, unbound
  20. USE_LZ_REDIRECT=0 # enable or not to use local-zone redirect for unbound
  21. RACINE="$(dirname "$(readlink -f -- "$0")")"
  22. DIR_DL="${RACINE}/downloads"
  23. DIR_LISTS="${RACINE}/lists"
  24. DIR_SRC="${RACINE}/src"
  25. list="domains"
  26. now="$(date +"%x %X")"
  27. today="$(date -d "${now}" +%s)"
  28. declare -i seconds=86400 # in seconds
  29. declare -a blocklists
  30. ARG="$1"
  31. if [ -z "${ARG}" ]; then ARG="unbound"; fi
  32. ###
  33. #
  34. # Functions
  35. #
  36. ###
  37. ### Get data into file in var list...
  38. build_blocklists() {
  39. printf "%s \n" "*** Manage list: ${DIR_SRC}/${list} ***"
  40. if [ -f "${DIR_SRC}/${list}" ]; then
  41. i=0
  42. while read -r line; do
  43. #if [ "$(echo "${line}" | grep -v "^#")" ]; then blocklists[$i]="${line}"; fi
  44. if echo "${line}" | grep -v "^#"; then blocklists[$i]="${line}"; fi
  45. let i++
  46. done < "${DIR_SRC}/${list}"
  47. unset i
  48. else
  49. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "Le fichier ${DIR_SRC}/${list} semble ne pas être lisible !"
  50. exit 1
  51. fi
  52. }
  53. build_sums() {
  54. if [ -f "${DIR_LISTS}/${output}" ]; then
  55. cd "${DIR_LISTS}" || exit 1
  56. if sha512sum --tag "${output}" > "${output}.sha512"; then
  57. printf "[ \\33[1;32m%s\\33[0;39m ] %s \n" "OK" "The checksum file is correctly created!"
  58. else
  59. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "It seems to have a problem to create checksum file!"
  60. fi
  61. cd "${RACINE}" || exit 1
  62. fi
  63. }
  64. build_uniq_list() {
  65. # on s'assure d'une liste de noms uniques
  66. ## http://promberger.info/linux/2009/01/14/removing-duplicate-lines-from-a-file/
  67. #{ rm "${DIR_SRC}/uniq_${list}" && awk '!x[tolower($1)]++' > "${DIR_SRC}/uniq_${list}"; } < "${DIR_SRC}/uniq_${list}"
  68. # shellcheck disable=SC2094
  69. { rm "${DIR_SRC}/uniq_${list}" && awk '!x[tolower($1)]++' | sort -du -o "${DIR_SRC}/uniq_${list}"; } < "${DIR_SRC}/uniq_${list}"
  70. }
  71. download() {
  72. printf "%s \n" "########## Attempt to get blocklists files ##########"
  73. printf "%s \n" "=> Attempt to download file: ${filename}"
  74. local bool=0
  75. [ ! -d "${DIR_DL}" ] && mkdir "${DIR_DL}"
  76. if [ -x "$(which curl)" ]; then
  77. if ! "$(which curl)" -A "Mozilla/5.0 (X11; Debian; Linux; rv:0.0) Gecko/20100101 Firefox/0.0" -o "${filename}" "${url}"; then
  78. bool=1
  79. fi
  80. elif [ -x "$(which wget)" ]; then
  81. if ! "$(which wget)" --user-agent="Mozilla/5.0 (X11; Debian; Linux; rv:0.0) Gecko/20100101 Firefox/0.0" -O "${filename}" "${url}"; then
  82. bool=1
  83. fi
  84. fi
  85. if [ ${bool} -eq 0 ]; then
  86. printf "[ \\33[0;32m%s\\33[0;39m ] %s \n" "OK" "The file ${filename} is correctly downloaded!"
  87. else
  88. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "It seems to have a problem with download file ${filename}!"
  89. fi
  90. }
  91. del_uniq_list() {
  92. if [ -f "${DIR_SRC}/uniq_${list}" ]; then rm "${DIR_SRC}/uniq_${list}"; fi
  93. }
  94. make_uniq_list() {
  95. local mime=""
  96. if [ ! -f "${DIR_SRC}/uniq_${list}" ]; then touch "${DIR_SRC}/uniq_${list}"; fi
  97. #mime="text/plain; charset=us-ascii"
  98. #case "${ndd}" in
  99. #"mirror1.malwaredomains.com")
  100. #case "${file}" in
  101. #"immortal_domains.zip")
  102. #mime="text/x-c++; charset=us-ascii"
  103. #;;
  104. #"malwaredomains.zones.zip")
  105. #mime="text/plain; charset=us-ascii"
  106. #;;
  107. #esac
  108. #;;
  109. #"winhelp2002.mvps.org")
  110. #mime="text/x-c; charset=us-ascii"
  111. #;;
  112. #*)
  113. #mime="text/plain; charset=us-ascii"
  114. #;;
  115. #esac
  116. #if [ "$(file -b -i "${filename}")" = "${mime}" ]; then
  117. printf "%s \n" "====> Attempt to make uniq file with filename: ${filename}"
  118. # !a[$0]++
  119. awk '{ print tolower($0) }' "${filename}" >> "${DIR_SRC}/uniq_${list}"
  120. #awk '!a[tolower($0)]++' "${filename}" >> "${DIR_SRC}/uniq_${list}"
  121. #fi
  122. unset mime
  123. }
  124. # Create uniq list file by datas into array blocklist
  125. mng_blocklists() {
  126. printf "%s \n" "### Essai de lecture des données blocklist"
  127. count="${#blocklists[@]}"
  128. if [ "${count}" -gt 0 ]; then
  129. for url in "${blocklists[@]}"; do
  130. if [ "${url}" = "personals" ]; then
  131. filename="${DIR_SRC}/${url}"
  132. else
  133. ndd="$(echo "${url}" | awk -F'/' '{ print $3 }')"
  134. file="$(echo "${url##*/}" | sed -e 's#\?#_#g;s#=#_#g;s#php#txt#g;s#\&#_#g')";
  135. name="${ndd}_${file}"
  136. filename="${DIR_DL}/${name}"
  137. fi
  138. #printf "file: %s \n" "${file}"
  139. # define seconds before new dl
  140. case "${ndd}" in
  141. "mirror1.malwaredomains.com") seconds=2592000;; # 1 month
  142. "winhelp2002.mvps.org") seconds=604800;; # 7 days
  143. "www.spamhaus.org") seconds=3600;; # 1 hours
  144. #*) seconds=86400;;
  145. esac
  146. if [ -f "${filename}" ]; then
  147. # get file seconds stat
  148. if [ -x "$(which stat)" ]; then
  149. file_seconds=$(stat -c "%Y" "${filename}")
  150. else
  151. file_seconds=$(date -r "${filename}" +%s)
  152. fi
  153. #today="$(date -d "${now}" +%s)"
  154. # calcul diff time in seconds
  155. if [ -x "$(which bc)" ]; then
  156. diff_sec=$(echo "${today} - ${file_seconds}" | bc)
  157. else
  158. diff_sec=$((today - file_seconds))
  159. fi
  160. unset file_seconds today
  161. if [ ${diff_sec} -gt ${seconds} ]; then download; fi
  162. else
  163. download
  164. fi
  165. uncompress
  166. case "${ndd}" in
  167. "hosts-file.net")
  168. if [ "$(file -b -i "${filename}")" = "application/zip; charset=binary" ]; then
  169. filename="${filename%.zip}/hosts.txt"
  170. fi
  171. ;;
  172. "mirror1.malwaredomains.com")
  173. if [ "$(file -b -i "${filename}")" = "application/zip; charset=binary" ]; then
  174. case "${file}" in
  175. "immortal_domains.zip")
  176. filename="${filename%.zip}/${file%.zip}.txt"
  177. ;;
  178. "justdomains.zip"|"malwaredomains.zones.zip")
  179. filename="${filename%.zip}/${file%.zip}"
  180. ;;
  181. esac
  182. fi
  183. ;;
  184. "winhelp2002.mvps.org")
  185. if [ "$(file -b -i "${filename}")" = "application/zip; charset=binary" ]; then
  186. filename="${filename%.zip}/HOSTS"
  187. fi
  188. ;;
  189. esac
  190. purge_files
  191. make_uniq_list
  192. unset filename
  193. done
  194. else
  195. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "Il semble ne pas y avoir de données récupérées !"
  196. exit 1
  197. fi
  198. unset count
  199. }
  200. purge_files() {
  201. printf "%s \n" "===> Attempt to transform downloaded file: ${filename}"
  202. # /^$/d <= empty line
  203. # /^#/d <= line starting with #
  204. # s/ \+//g <= remove all spaces
  205. # s/ \+/ /g <= replace more spaces by only one
  206. case "${ndd}" in
  207. "adaway.org")
  208. case "${ARG}" in
  209. "host"|"hosts"|"host0")
  210. sed -i -e "/^$/d;/^#/d;s/\(.*\) ##\(.*\)#/\1/g;/\(.*\)localhost\(.*\)/d;" "${filename}"
  211. ;;
  212. *)
  213. sed -i -e "/^$/d;/^#/d;s/\(.*\) ##\(.*\)#/\1/g;/\(.*\)localhost\(.*\)/d;s#127.0.0.1\ \(.*\)#\1#g;" "${filename}"
  214. ;;
  215. esac
  216. ;;
  217. "hosts-file.net")
  218. case "${ARG}" in
  219. "host"|"hosts"|"host0")
  220. sed -i -e "/^$/d;/^#/d;/\(.*\)localhost\(.*\)/d;s/ \+//g" "${filename}"
  221. ;;
  222. *)
  223. sed -i -e "/^$/d;/^#/d;/\(.*\)localhost\(.*\)/d;s#127.0.0.1\t\(.*\)#\1#g;s/ \+//g" "${filename}"
  224. ;;
  225. esac
  226. ;;
  227. "mirror1.malwaredomains.com")
  228. case "${file}" in
  229. "immortal_domains.zip")
  230. sed -i -e "/^#/d;/^notice/d;s/ \+//g" "${filename}"
  231. ;;
  232. "malwaredomains.zones.zip")
  233. sed -i -e "/^\/\//d;s/ \+/ /g;s#zone \"\(.*\)\" {type master; file \"/etc/namedb/blockeddomain.hosts\";};#\1#g" "${filename}"
  234. ;;
  235. esac
  236. ;;
  237. "someonewhocares.org")
  238. case "${ARG}" in
  239. "host"|"hosts"|"host0")
  240. sed -i -e "/^#/d;/^\t\+#/d;s/^[ \t]*//g;/^ \+#/d;/\(.*\)local\(.*\)/d;/\(.*\)broadcast\(.*\)/d;s/\(.*\)#\(.*\)/\1/g;/^$/d" "${filename}"
  241. ;;
  242. *)
  243. sed -i -e "/^#/d;/^\t\+#/d;s/^[ \t]*//g;/^ \+#/d;/\(.*\)local\(.*\)/d;/\(.*\)broadcast\(.*\)/d;s/\(.*\)#\(.*\)/\1/g;/^$/d;s#127.0.0.1 \(.*\)#\1#g" "${filename}"
  244. ;;
  245. esac
  246. ;;
  247. "winhelp2002.mvps.org")
  248. case "${ARG}" in
  249. "host0")
  250. sed -i -e "/^#/d;/^$/d;/^\r/d;/\(.*\)localhost\(.*\)/d;s/\(.*\)#\(.*\)/\1/g;s/ \+//g" "${filename}"
  251. ;;
  252. *)
  253. sed -i -e "/^#/d;/^$/d;/^\r/d;/\(.*\)localhost\(.*\)/d;s#0.0.0.0 \(.*\)#\1#g;s/\(.*\)#\(.*\)/\1/g;s/ \+//g" "${filename}"
  254. ;;
  255. esac
  256. ;;
  257. "www.malwaredomainlist.com")
  258. case "${ARG}" in
  259. "host"|"hosts"|"host0")
  260. sed -i -e "/^\r/d;/^#/d;/\(.*\)localhost\(.*\)/d;s/ \+//g" "${filename}"
  261. ;;
  262. *)
  263. sed -i -e "/^\r/d;/^#/d;/\(.*\)localhost\(.*\)/d;s#127.0.0.1\(.*\)#\1#g;s/ \+//g" "${filename}"
  264. ;;
  265. esac
  266. ;;
  267. *) # delete empty lines, ^#, keep all others :p
  268. sed -i -e "/^$/d;/^#/d;s/ \+//g" "${filename}"
  269. ;;
  270. esac
  271. }
  272. transformer() {
  273. printf "%s \n" "===> Attempt to transform unic list '${DIR_SRC}/uniq_${list}' in list '${ARG}'..."
  274. case "${ARG}" in
  275. "bind"|"bind8"|"bind9")
  276. format="Bind Config"
  277. output="bind.zone"
  278. ;;
  279. "host"|"hosts"|"host0")
  280. format="hosts"
  281. output="hosts"
  282. ;;
  283. "unbound")
  284. format="Local zone"
  285. output="local-zone"
  286. ;;
  287. "pf")
  288. format="Packet-Filter"
  289. output="baddomains"
  290. ;;
  291. esac
  292. if [ -f "${DIR_SRC}/uniq_${list}" ]; then
  293. mssg="###########################
  294. ### Block Zones Project ###
  295. ###########################
  296. ### Format '${format}'
  297. ### Date : ${now}
  298. ##
  299. #"
  300. echo "${mssg}" > "${DIR_LISTS}/${output}"
  301. unset mssg
  302. case "${ARG}" in
  303. "host0")
  304. echo "0.0.0.0 localhost" >> "${DIR_LISTS}/${output}"
  305. ;;
  306. "host"|"hosts")
  307. if [ "${IPv4}" = 1 ]; then echo "127.0.0.1 localhost" >> "${DIR_LISTS}/${output}"; fi
  308. if [ "${IPv6}" = 1 ]; then echo "::1 localhost" >> "${DIR_LISTS}/${output}"; fi
  309. ;;
  310. esac
  311. i=0
  312. while read -r line; do
  313. line="${line//[$'\t\r\n ']}"
  314. case "${ARG}" in
  315. "bind8")
  316. echo "zone \"${line}\" { type master; notify no; file \"null.zone.file\"; };" >> "${DIR_LISTS}/${output}"
  317. ;;
  318. "bind"|"bind9")
  319. echo "zone \"${line}\" { type master; notify no; file \"/etc/bind/nullzonefile.txt\"; };" >> "${DIR_LISTS}/${output}"
  320. ;;
  321. "host0")
  322. echo "0.0.0.0 ${line}" >> "${DIR_LISTS}/${output}"
  323. ;;
  324. "host"|"hosts")
  325. if [ "${IPv4}" = 1 ]; then echo "127.0.0.1 ${line}" >> "${DIR_LISTS}/${output}"; fi
  326. if [ "${IPv6}" = 1 ]; then echo "::1 ${line}" >> "${DIR_LISTS}/${output}"; fi
  327. ;;
  328. "pf")
  329. echo "${line}" >> "${DIR_LISTS}/${output}"
  330. ;;
  331. "unbound")
  332. if [ "${IPv4}" = 1 ]; then
  333. if [ "${USE_LZ_REDIRECT}" = 1 ]; then
  334. echo -e "local-zone: \"${line}\" redirect\nlocal-data: \"${line} A 127.0.0.1\"" >> "${DIR_LISTS}/${output}";
  335. else
  336. echo -e "local-data: \"${line} A 127.0.0.1\"" >> "${DIR_LISTS}/${output}";
  337. fi
  338. fi
  339. if [ "${IPv6}" = 1 ]; then
  340. if [ "${USE_LZ_REDIRECT}" = 1 ]; then
  341. echo -e "local-zone: \"${line}\" redirect\nlocal-data: \"${line} AAAA ::1\"" >> "${DIR_LISTS}/${output}";
  342. else
  343. echo -e "local-data: \"${line} AAAA ::1\"" >> "${DIR_LISTS}/${output}";
  344. fi
  345. fi
  346. ;;
  347. esac
  348. let i++
  349. done < "${DIR_SRC}/uniq_${list}"
  350. unset i
  351. else
  352. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "Il semble ne pas y avoir de fichier '${DIR_SRC}/uniq_${list}' !"
  353. exit 1
  354. fi
  355. unset arg
  356. if [ -f "${DIR_LISTS}/${output}" ]; then
  357. printf "[ \\33[1;32m%s\\33[0;39m ] %s \n" "OK" "Le fichier '${DIR_LISTS}/${output}' semble avoir été construit !"
  358. build_sums
  359. else
  360. printf "[ \\33[1;31m%s\\33[0;39m ] %s \n" "KO" "Il semble ne pas y avoir de fichier '${DIR_LISTS}/${output}' !"
  361. exit 1
  362. fi
  363. }
  364. uncompress() {
  365. if [ "$(file -b -i "${filename}")" = "application/gzip; charset=binary" ]; then
  366. printf "%s \n" "==> Attempt to extract archive .gz: ${filename}"
  367. gunzip -d -f -q "${filename}";
  368. fi
  369. if [ "$(file -b -i "${filename}")" = "application/zip; charset=binary" ]; then
  370. printf "%s \n" "==> Attempt to extract archive .zip: ${filename}"
  371. unzip -oqu "${filename}" -d "${filename%.zip}"
  372. fi
  373. }
  374. #verify_uid
  375. #verify_need_dirs
  376. del_uniq_list
  377. build_blocklists
  378. mng_blocklists
  379. build_uniq_list
  380. transformer