Merge pull request #574 from six2dez/dev

2.4 release

Author: six2dez

README.md

@@ -8,8 +8,8 @@
 
 
 <p align="center">
-  <a href="https://github.com/six2dez/reconftw/releases/tag/v2.3.2">
-    <img src="https://img.shields.io/badge/release-v2.3.2-green">
+  <a href="https://github.com/six2dez/reconftw/releases/tag/v2.4">
+    <img src="https://img.shields.io/badge/release-v2.4-green">
   </a>
    </a>
   <a href="https://www.gnu.org/licenses/gpl-3.0.en.html">
@@ -440,6 +440,7 @@ reset='\033[0m'
 ## Subdomains
   - Passive ([amass](https://github.com/OWASP/Amass) and [github-subdomains](https://github.com/gwen001/github-subdomains))
   - Certificate transparency ([ctfr](https://github.com/UnaPibaGeek/ctfr))
+  - NOERROR subdomain discovery ([dnsx](https://github.com/projectdiscovery/dnsx), more info [here](https://www.securesystems.de/blog/enhancing-subdomain-enumeration-ents-and-noerror/))
   - Bruteforce ([puredns](https://github.com/d3mondev/puredns))
   - Permutations ([Gotator](https://github.com/Josue87/gotator))
   - JS files & Source Code Scraping ([gospider](https://github.com/jaeles-project/gospider))
@@ -465,7 +466,7 @@ reset='\033[0m'
 - Web screenshot ([webscreenshot](https://github.com/maaaaz/webscreenshot) or [gowitness](https://github.com/sensepost/gowitness))
 - Web templates scanner ([nuclei](https://github.com/projectdiscovery/nuclei) and [nuclei geeknik](https://github.com/geeknik/the-nuclei-templates.git))
 - Url extraction ([waybackurls](https://github.com/tomnomnom/waybackurls), [gau](https://github.com/lc/gau), [gospider](https://github.com/jaeles-project/gospider), [github-endpoints](https://gist.github.com/six2dez/d1d516b606557526e9a78d7dd49cacd3) and [JSA](https://github.com/w9w/JSA))
-- URLPatterns Search ([gf](https://github.com/tomnomnom/gf) and [gf-patterns](https://github.com/1ndianl33t/Gf-Patterns))
+- URLPatterns Search and filtering ([urless](https://github.com/xnl-h4ck3r/urless), [gf](https://github.com/tomnomnom/gf) and [gf-patterns](https://github.com/1ndianl33t/Gf-Patterns))
 - XSS ([dalfox](https://github.com/hahwul/dalfox))
 - Open redirect ([Oralyzer](https://github.com/r0075h3ll/Oralyzer))
 - SSRF (headers [interactsh](https://github.com/projectdiscovery/interactsh) and param values with [ffuf](https://github.com/ffuf/ffuf))
@@ -486,7 +487,7 @@ reset='\033[0m'
 - Passwords dictionary creation ([pydictor](https://github.com/LandGrey/pydictor))
 
 ## Extras
-- Multithread ([Interlace](https://github.com/codingo/Interlace))
+- Multithread ([Rush](https://github.com/shenwei356/rush))
 - Custom resolvers generated list ([dnsvalidator](https://github.com/vortexau/dnsvalidator))
 - Docker container included and [DockerHub](https://hub.docker.com/r/six2dez/reconftw) integration
 - Ansible + Terraform deployment over AWS
@@ -498,7 +499,7 @@ reset='\033[0m'
 - Support for targets with multiple domains
 - Raspberry Pi/ARM support
 - 6 modes (recon, passive, subdomains, web, osint and all)
-- Out of Scope Support
+- Out of Scope Support + optional [inscope](https://github.com/tomnomnom/hacks/tree/master/inscope) support
 - Notification system with Slack, Discord and Telegram ([notify](https://github.com/projectdiscovery/notify)) and sending zipped results support
 
 # Mindmap/Workflow
@@ -549,18 +550,16 @@ If you want to contribute to this project you can do it in multiple ways:
 **This section shows the current financial sponsors of this project**
 
 [<img src="https://pbs.twimg.com/profile_images/1360304248534282240/MomOFi40_400x400.jpg" width="100" height=auto>](https://github.com/0xtavian)
-[<img src="https://pbs.twimg.com/profile_images/1509304017993752578/qqcKUKZb_400x400.jpg" width="100" height=auto>](https://github.com/geeknik)
 
 # Thanks :pray:
 * Thank you for lending a helping hand towards the development of the project!
 
-- [Spyse](https://spyse.com/)
-- [Networksdb](https://networksdb.io/)
-- [Intelx](https://intelx.io/)
-- [BinaryEdge](https://www.binaryedge.io/)
-- [Censys](https://censys.io/)
+- [C99](https://api.c99.nl/)
 - [CIRCL](https://www.circl.lu/)
-- [Whoxy](https://www.whoxy.com/)
+- [NetworksDB](networksdb.io)
+- [ipinfo](ipinfo.io)
+- [hackertarget](hackertarget.com)
+- [Censys](censys.io)
 
 # Disclaimer
 Usage of this program for attacking targets without consent is illegal. It is the user's responsibility to obey all applicable laws. The developer assumes no liability and is not responsible for any misuse or damage caused by this program. Please use responsibly.

install.sh

@@ -76,6 +76,9 @@ gotools["tlsx"]="go install github.com/projectdiscovery/tlsx/cmd/tlsx@latest"
 gotools["gitdorks_go"]="go install -v github.com/damit5/gitdorks_go@latest"
 gotools["smap"]="go install -v github.com/s0md3v/smap/cmd/smap@latest"
 gotools["dsieve"]="go install -v github.com/trickest/dsieve@master"
+gotools["inscope"]="go install github.com/tomnomnom/hacks/inscope@latest"
+gotools["rush"]="go install github.com/shenwei356/rush@latest"
+gotools["enumerepo"]="go install github.com/trickest/enumerepo@latest"
 
 declare -A repos
 repos["dorks_hunter"]="six2dez/dorks_hunter"
@@ -91,7 +94,6 @@ repos["xnLinkFinder"]="xnl-h4ck3r/xnLinkFinder"
 repos["Corsy"]="s0md3v/Corsy"
 repos["CMSeeK"]="Tuhinshubhra/CMSeeK"
 repos["fav-up"]="pielco11/fav-up"
-repos["Interlace"]="codingo/Interlace"
 repos["massdns"]="blechschmidt/massdns"
 repos["Oralyzer"]="r0075h3ll/Oralyzer"
 repos["testssl"]="drwetter/testssl.sh"
@@ -101,6 +103,8 @@ repos["cloud_enum"]="initstring/cloud_enum"
 repos["ultimate-nmap-parser"]="shifty0g/ultimate-nmap-parser"
 repos["pydictor"]="LandGrey/pydictor"
 repos["gitdorks_go"]="damit5/gitdorks_go"
+repos["urless"]="xnl-h4ck3r/urless"
+repos["trufflehog"]="trufflesecurity/trufflehog"
 
 printf "\n\n${bgreen}#######################################################################${reset}\n"
 printf "${bgreen} reconFTW installer/updater script ${reset}\n\n"
@@ -312,11 +316,13 @@ for repo in "${!repos[@]}"; do
         eval $SUDO pip3 install . $DEBUG_STD
     fi
     if [ "massdns" = "$repo" ]; then
-            eval make $DEBUG_STD && strip -s bin/massdns && eval $SUDO cp bin/massdns /usr/local/bin/ $DEBUG_ERROR
+        eval make $DEBUG_STD && strip -s bin/massdns && eval $SUDO cp bin/massdns /usr/local/bin/ $DEBUG_ERROR
     elif [ "gf" = "$repo" ]; then
-            eval cp -r examples ~/.gf $DEBUG_ERROR
+        eval cp -r examples ~/.gf $DEBUG_ERROR
     elif [ "Gf-Patterns" = "$repo" ]; then
-            eval mv *.json ~/.gf $DEBUG_ERROR
+        eval mv *.json ~/.gf $DEBUG_ERROR
+    elif [ "trufflehog" = "$repo" ]; then
+        go install
     fi
     cd "$dir" || { echo "Failed to cd to $dir in ${FUNCNAME[0]} @ line ${LINENO}"; exit 1; }
 done

reconftw.cfg

@@ -40,6 +40,7 @@ DEBUG_ERROR="2>/dev/null" # Skips ERR output on installer
 OSINT=true # Enable or disable the whole OSINT module
 GOOGLE_DORKS=true
 GITHUB_DORKS=true
+GITHUB_REPOS=true
 METADATA=true # Fetch metadata from indexed office documents
 EMAILS=true # Fetch emails from differents sites 
 DOMAIN_INFO=true # whois info
@@ -51,25 +52,29 @@ METAFINDER_LIMIT=20 # Max 250
 SUBDOMAINS_GENERAL=true # Enable or disable the whole Subdomains module
 SUBPASSIVE=true # Passive subdomains search
 SUBCRT=true # crtsh search
+SUBNOERROR=true # Check DNS NOERROR response and BF on them
 SUBANALYTICS=true # Google Analytics search
 SUBBRUTE=true # DNS bruteforcing
 SUBSCRAPING=true # Subdomains extraction from web crawling
 SUBPERMUTE=true # DNS permutations
 PERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper)
+GOTATOR_FLAGS="-depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator
 SUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check this
 SUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queries
 DEEP_RECURSIVE_PASSIVE=10 # Number of top subdomains for recursion
 SUB_RECURSIVE_BRUTE=false # Needs big disk space and time to resolve
 ZONETRANSFER=true # Check zone transfer
 S3BUCKETS=true # Check S3 buckets misconfigs
 REVERSE_IP=false # Check reverse IP subdomain search (set True if your target is CIDR/IP)
-TLS_PORTS="21,22,25,80,110,135,143,261,271,324,443,448,465,563,614,631,636,664,684,695,832,853,854,990,993,989,990,992,993,994,995,1129,1131,1184,2083,2087,2089,2096,2221,2252,2376,2381,2478,2479,2482,2484,2679,2762,3077,3078,3183,3191,3220,3269,3306,3410,3424,3471,3496,3509,3529,3539,3535,3660,36611,3713,3747,3766,3864,3885,3995,3896,4031,4036,4062,4064,4081,4083,4116,4335,4336,4536,4590,4740,4843,4843,4849,5443,5007,5061,5321,5349,5671,5783,5868,5986,5989,5990,6209,6251,6443,6513,6514,6619,6697,6771,6697,7202,7443,7673,7674,7677,7775,8243,8443,8991,8989,9089,9295,9318,9443,9444,9614,9802,10161,10162,11751,12013,12109,14143,15002,16995,41230,16993,20003"
+TLS_PORTS="21,22,25,80,110,135,143,261,271,324,443,448,465,563,614,631,636,664,684,695,832,853,854,990,993,989,992,994,995,1129,1131,1184,2083,2087,2089,2096,2221,2252,2376,2381,2478,2479,2482,2484,2679,2762,3077,3078,3183,3191,3220,3269,3306,3410,3424,3471,3496,3509,3529,3539,3535,3660,36611,3713,3747,3766,3864,3885,3995,3896,4031,4036,4062,4064,4081,4083,4116,4335,4336,4536,4590,4740,4843,4849,5443,5007,5061,5321,5349,5671,5783,5868,5986,5989,5990,6209,6251,6443,6513,6514,6619,6697,6771,7202,7443,7673,7674,7677,7775,8243,8443,8991,8989,9089,9295,9318,9443,9444,9614,9802,10161,10162,11751,12013,12109,14143,15002,16995,41230,16993,20003"
+INSCOPE=false # Uses inscope tool to filter the scope, requires .scope file in reconftw folder 
 
 # Web detection
 WEBPROBESIMPLE=true # Web probing on 80/443
 WEBPROBEFULL=true # Web probing in a large port list
 WEBSCREENSHOT=true # Webs screenshooting
 VIRTUALHOSTS=false # Check virtualhosts by fuzzing HOST header
+NMAP_WEBPROBE=true # If disabled it will run httpx directly over subdomains list, nmap before web probing is used to increase the speed and avoid repeated requests
 UNCOMMON_PORTS_WEB="81,300,591,593,832,981,1010,1311,1099,2082,2095,2096,2480,3000,3128,3333,4243,4567,4711,4712,4993,5000,5104,5108,5280,5281,5601,5800,6543,7000,7001,7396,7474,8000,8001,8008,8014,8042,8060,8069,8080,8081,8083,8088,8090,8091,8095,8118,8123,8172,8181,8222,8243,8280,8281,8333,8337,8443,8500,8834,8880,8888,8983,9000,9001,9043,9060,9080,9090,9091,9092,9200,9443,9502,9800,9981,10000,10250,11371,12443,15672,16080,17778,18091,18092,20720,32000,55440,55672"
 # You can change to aquatone if gowitness fails, comment the one you don't want
 AXIOM_SCREENSHOT_MODULE=webscreenshot # Choose between aquatone,gowitness,webscreenshot
@@ -169,7 +174,7 @@ CMSSCAN_TIMEOUT=3600            # Seconds
 FFUF_MAXTIME=900                # Seconds
 HTTPX_TIMEOUT=10                # Seconds
 HTTPX_UNCOMMONPORTS_TIMEOUT=10  # Seconds
-GOTATOR_TIMEOUT="timeout 480"   # Manual timeout, minutes
+PERMUTATIONS_LIMIT=21474836480  # Bytes, default is 20 GB
 
 # lists
 fuzz_wordlist=${tools}/fuzz_wordlist.txt