Skip to content

Commit e261ee8

Browse files
Mzack9999Mzack9999
authored
Merge pull request #705 from projectdiscovery/dev
v1.2.0
2 parents 4f9bcff + 0ac7391 commit e261ee8

32 files changed

+984
-273
lines changed

‎.github/workflows/build-test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ jobs:
1313
runs-on: ${{ matrix.os }}
1414
strategy:
1515
matrix:
16-
os: [ubuntu-latest, windows-latest, macOS-13]
17-
go-version: [1.22.x]
16+
os: [ubuntu-latest, windows-latest, macOS-latest]
17+
go-version: [1.24.x]
1818
steps:
1919
- name: Set up Go
2020
uses: actions/setup-go@v4

‎.github/workflows/provider-integration.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ jobs:
1313
- name: Set up Go
1414
uses: actions/setup-go@v4
1515
with:
16-
go-version: 1.22.x
16+
go-version: 1.24.x
1717

1818
- name: Check out code
1919
uses: actions/checkout@v3
2020

2121
- name: Integration Tests
2222
env:
2323
GH_ACTION: true
24-
CENSYS_API_KEY: ${{secrets.CENSYS_API_SECRET}}
24+
CENSYS_API_KEY: ${{secrets.CENSYS_ORGANIZATION_ID}}
2525
FOFA_API_KEY: ${{secrets.FOFA_KEY}}
2626
SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
2727
ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}}

‎.github/workflows/release-binary.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
- name: "Set up Go"
1919
uses: actions/setup-go@v4
2020
with:
21-
go-version: 1.22.x
21+
go-version: 1.24.x
2222

2323
- name: "Create release on GitHub"
2424
uses: goreleaser/goreleaser-action@v4

‎.github/workflows/release-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
- name: Set up Go
2121
uses: actions/setup-go@v4
2222
with:
23-
go-version: 1.22.x
23+
go-version: 1.24.x
2424

2525
- name: release test
2626
uses: goreleaser/goreleaser-action@v4

‎README.md‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,8 @@ shodan:
135135
- SHODAN_API_KEY_1
136136
- SHODAN_API_KEY_2
137137
censys:
138-
- CENSYS_API_ID_1:CENSYS_API_SECRET_1
139-
- CENSYS_API_ID_2:CENSYS_API_SECRET_2
138+
- CENSYS_API_TOKEN_1:CENSYS_ORGANIZATION_ID_1
139+
- CENSYS_API_TOKEN_2:CENSYS_ORGANIZATION_ID_2
140140
fofa:
141141
- FOFA_EMAIL_1:FOFA_KEY_1
142142
- FOFA_EMAIL_2:FOFA_KEY_2
@@ -178,8 +178,8 @@ alternatively you can also set the API key as environment variable in your bash
178178
179179
```yaml
180180
export SHODAN_API_KEY=xxx
181-
export CENSYS_API_ID=xxx
182-
export CENSYS_API_SECRET=xxx
181+
export CENSYS_API_TOKEN=xxx
182+
export CENSYS_ORGANIZATION_ID=xxx
183183
export FOFA_EMAIL=xxx
184184
export FOFA_KEY=xxx
185185
export QUAKE_TOKEN=xxx
@@ -195,15 +195,7 @@ export ONYPHE_API_KEY=xxx
195195
export DRIFTNET_API_KEY=xxx
196196
```
197197

198-
Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), ZoomEye [china](https://api.zoomeye.org) - [worldwide](https://api.zoomeye.hk), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup) and [Driftnet](https://driftnet.io/auth?state=signup).
199-
200-
### ZoomEye API
201-
202-
Before conducting any scans, please ensure you are using the correct host to comply with geographical access restrictions of the ZoomEye API:
203-
204-
- **zoomeye.org** is exclusively for users within China.
205-
- **zoomeye.hk** is for users outside China (this is default if no host provided).
206-
198+
Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.ai), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup) and [Driftnet](https://driftnet.io/auth?state=signup).
207199

208200
## Running Uncover
209201

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/projectdiscovery/uncover
33
go 1.24.0
44

55
require (
6+
github.com/censys/censys-sdk-go v0.19.1
67
github.com/hashicorp/golang-lru v0.5.4
78
github.com/julienschmidt/httprouter v1.3.0
89
github.com/logrusorgru/aurora v2.0.3+incompatible
@@ -63,6 +64,7 @@ require (
6364
github.com/dlclark/regexp2 v1.11.4 // indirect
6465
github.com/docker/go-units v0.5.0 // indirect
6566
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
67+
github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05 // indirect
6668
github.com/fatih/color v1.15.0 // indirect
6769
github.com/gaissmai/bart v0.17.8 // indirect
6870
github.com/go-ole/go-ole v1.2.6 // indirect

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6
3434
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
3535
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
3636
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
37+
github.com/censys/censys-sdk-go v0.19.1 h1:CG8rQKgwrKuoICd3oU0uddALMfJnboeMkDg/e74HYyc=
38+
github.com/censys/censys-sdk-go v0.19.1/go.mod h1:DgPz5NgL+EfoueXLPG9UG1e7hS0OhtlywgpkIuu3ZRE=
3739
github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
3840
github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
3941
github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw=
@@ -62,6 +64,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
6264
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
6365
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
6466
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
67+
github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05 h1:S92OBrGuLLZsyM5ybUzgc/mPjIYk2AZqufieooe98uw=
68+
github.com/ericlagergren/decimal v0.0.0-20221120152707-495c53812d05/go.mod h1:M9R1FoZ3y//hwwnJtO51ypFGwm8ZfpxPT/ZLtO1mcgQ=
6569
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
6670
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
6771
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

‎integration-tests/integration-test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"strings"
77

88
"github.com/logrusorgru/aurora"
9-
109
"github.com/projectdiscovery/uncover/testutils"
1110
)
1211

@@ -33,6 +32,7 @@ var (
3332
"odin": odinTestcases{},
3433
"binaryedge": binaryedgeTestcases{},
3534
"onyphe": onypheTestcases{},
35+
"greynoise": greynoiseTestcases{},
3636
// feature tests
3737
"output": outputTestcases{},
3838
}

‎integration-tests/source-test.go‎

Lines changed: 64 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ func (h censysTestcases) Execute() error {
2323
}
2424
censysToken := fmt.Sprintf(`censys: [%s]`, token)
2525
_ = os.WriteFile(ConfigFile, []byte(censysToken), 0644)
26-
defer os.RemoveAll(ConfigFile)
26+
defer func() {
27+
_ = os.RemoveAll(ConfigFile)
28+
}()
2729
results, err := testutils.RunUncoverAndGetResults(debug, "-censys", "'services.software.vendor=Grafana'")
2830
if err != nil {
2931
return err
@@ -40,7 +42,9 @@ func (h shodanTestcases) Execute() error {
4042
}
4143
shodanToken := fmt.Sprintf(`shodan: [%s]`, token)
4244
_ = os.WriteFile(ConfigFile, []byte(shodanToken), 0644)
43-
defer os.RemoveAll(ConfigFile)
45+
defer func() {
46+
_ = os.RemoveAll(ConfigFile)
47+
}()
4448
results, err := testutils.RunUncoverAndGetResults(debug, "-shodan", "'title:\"Grafana\"'")
4549
if err != nil {
4650
return err
@@ -65,7 +69,9 @@ func (h zoomeyeTestcases) Execute() error {
6569
}
6670
zoomeyeToken := fmt.Sprintf(`zoomeye: [%s]`, token)
6771
_ = os.WriteFile(ConfigFile, []byte(zoomeyeToken), 0644)
68-
defer os.RemoveAll(ConfigFile)
72+
defer func() {
73+
_ = os.RemoveAll(ConfigFile)
74+
}()
6975
results, err := testutils.RunUncoverAndGetResults(debug, "-zoomeye", "'app:\"Atlassian JIRA\"'")
7076
if err != nil {
7177
return err
@@ -82,7 +88,9 @@ func (h fofaTestcases) Execute() error {
8288
}
8389
fofaToken := fmt.Sprintf(`fofa: [%s]`, token)
8490
_ = os.WriteFile(ConfigFile, []byte(fofaToken), 0644)
85-
defer os.RemoveAll(ConfigFile)
91+
defer func() {
92+
_ = os.RemoveAll(ConfigFile)
93+
}()
8694
results, err := testutils.RunUncoverAndGetResults(debug, "-fofa", "'app=Grafana'")
8795
if err != nil {
8896
return err
@@ -109,7 +117,9 @@ func (h quakeTestcases) Execute() error {
109117
}
110118
quakeToken := fmt.Sprintf(`quake: [%s]`, token)
111119
_ = os.WriteFile(ConfigFile, []byte(quakeToken), 0644)
112-
defer os.RemoveAll(ConfigFile)
120+
defer func() {
121+
_ = os.RemoveAll(ConfigFile)
122+
}()
113123
results, err := testutils.RunUncoverAndGetResults(debug, "-quake", "'Grafana'")
114124
if err != nil {
115125
return err
@@ -126,7 +136,9 @@ func (h netlasTestcases) Execute() error {
126136
}
127137
netlasToken := fmt.Sprintf(`netlas: [%s]`, token)
128138
_ = os.WriteFile(ConfigFile, []byte(netlasToken), 0644)
129-
defer os.RemoveAll(ConfigFile)
139+
defer func() {
140+
_ = os.RemoveAll(ConfigFile)
141+
}()
130142
results, err := testutils.RunUncoverAndGetResults(debug, "-netlas", "'Grafana'")
131143
if err != nil {
132144
return err
@@ -143,7 +155,9 @@ func (h criminalipTestcases) Execute() error {
143155
}
144156
criminalipToken := fmt.Sprintf(`criminalip: [%s]`, token)
145157
_ = os.WriteFile(ConfigFile, []byte(criminalipToken), 0644)
146-
defer os.RemoveAll(ConfigFile)
158+
defer func() {
159+
_ = os.RemoveAll(ConfigFile)
160+
}()
147161
results, err := testutils.RunUncoverAndGetResults(debug, "-criminalip", "'Grafana'")
148162
if err != nil {
149163
return err
@@ -160,7 +174,9 @@ func (h hunterhowTestcases) Execute() error {
160174
}
161175
hunterhowApiKey := fmt.Sprintf(`hunterhow: [%s]`, token)
162176
_ = os.WriteFile(ConfigFile, []byte(hunterhowApiKey), 0644)
163-
defer os.RemoveAll(ConfigFile)
177+
defer func() {
178+
_ = os.RemoveAll(ConfigFile)
179+
}()
164180
results, err := testutils.RunUncoverAndGetResults(debug, "-hunterhow", "'web.body=\"ElasticJob\"'")
165181
if err != nil {
166182
return err
@@ -199,7 +215,9 @@ func (h googleTestcases) Execute() error {
199215
}
200216
googleToken := fmt.Sprintf(`google: [%s,%s]`, token, engineId)
201217
_ = os.WriteFile(ConfigFile, []byte(googleToken), 0644)
202-
defer os.RemoveAll(ConfigFile)
218+
defer func() {
219+
_ = os.RemoveAll(ConfigFile)
220+
}()
203221
results, err := testutils.RunUncoverAndGetResults(debug, "-google", "site:*.hackerone.com")
204222
if err != nil {
205223
return err
@@ -218,7 +236,9 @@ func (h odinTestcases) Execute() error {
218236
if err := os.WriteFile(ConfigFile, []byte(odinToken), 0644); err != nil {
219237
return err
220238
}
221-
defer os.RemoveAll(ConfigFile)
239+
defer func() {
240+
_ = os.RemoveAll(ConfigFile)
241+
}()
222242
results, err := testutils.RunUncoverAndGetResults(debug, "-odin", "nginx")
223243

224244
if err != nil {
@@ -237,7 +257,9 @@ func (h binaryedgeTestcases) Execute() error {
237257

238258
binaryedgeToken := fmt.Sprintf(`binaryedge: [%s]`, token)
239259
_ = os.WriteFile(ConfigFile, []byte(binaryedgeToken), 0644)
240-
defer os.RemoveAll(ConfigFile)
260+
defer func() {
261+
_ = os.RemoveAll(ConfigFile)
262+
}()
241263
results, err := testutils.RunUncoverAndGetResults(debug, "-binaryedge", "1.1.1.1")
242264

243265
if err != nil {
@@ -256,11 +278,41 @@ func (h onypheTestcases) Execute() error {
256278

257279
onypheToken := fmt.Sprintf(`onyphe: [%s]`, token)
258280
_ = os.WriteFile(ConfigFile, []byte(onypheToken), 0644)
259-
defer os.RemoveAll(ConfigFile)
281+
defer func() {
282+
_ = os.RemoveAll(ConfigFile)
283+
}()
260284
results, err := testutils.RunUncoverAndGetResults(debug, "-onyphe", "google.com")
261285

262286
if err != nil {
263287
return err
264288
}
265289
return expectResultsGreaterThanCount(results, 0)
266290
}
291+
292+
type greynoiseTestcases struct{}
293+
294+
func (h greynoiseTestcases) Execute() error {
295+
token := os.Getenv("GREYNOISE_API_KEY")
296+
if token == "" {
297+
return errors.New("missing greynoise api key")
298+
}
299+
300+
greynoiseToken := fmt.Sprintf(`greynoise: [%s]`, token)
301+
_ = os.WriteFile(ConfigFile, []byte(greynoiseToken), 0644)
302+
defer os.RemoveAll(ConfigFile)
303+
304+
results, err := testutils.RunUncoverAndGetResults(debug, "-e", "greynoise", "-q", "tag:scanner")
305+
if err != nil {
306+
fmt.Fprintf(os.Stderr, "WARNING: greynoise query failed: %v\n", err)
307+
fmt.Fprintln(os.Stderr, "INFO: This may happen if you are using a Community API key. GNQL queries require an Enterprise API key.")
308+
return nil
309+
}
310+
311+
if len(results) == 0 {
312+
fmt.Fprintln(os.Stderr, "INFO: greynoise returned 0 results.")
313+
fmt.Fprintln(os.Stderr, "NOTE: Community API keys cannot access GNQL queries. An Enterprise API key is required for this test to return data.")
314+
return nil
315+
}
316+
317+
return nil
318+
}

‎runner/options.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ type Options struct {
6262
BinaryEdge goflags.StringSlice
6363
Onyphe goflags.StringSlice
6464
Driftnet goflags.StringSlice
65+
GreyNoise goflags.StringSlice
6566
DisableUpdateCheck bool
6667
}
6768

@@ -73,7 +74,7 @@ func ParseOptions() *Options {
7374

7475
flagSet.CreateGroup("input", "Input",
7576
flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
76-
flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet) (default shodan)", goflags.FileNormalizedStringSliceOptions),
77+
flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise) (default shodan)", goflags.FileNormalizedStringSliceOptions),
7778
flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions),
7879
)
7980

@@ -94,6 +95,7 @@ func ParseOptions() *Options {
9495
flagSet.StringSliceVarP(&options.BinaryEdge, "binaryedge", "be", nil, "search query for binaryedge (example: -binaryedge 'query.txt')", goflags.FileStringSliceOptions),
9596
flagSet.StringSliceVarP(&options.Onyphe, "onyphe", "on", nil, "search query for onyphe (example: -onyphe 'query.txt')", goflags.FileStringSliceOptions),
9697
flagSet.StringSliceVarP(&options.Driftnet, "driftnet", "df", nil, "search query for driftnet (example: -driftnet 'query.txt')", goflags.FileStringSliceOptions),
98+
flagSet.StringSliceVarP(&options.GreyNoise, "greynoise", "gn", nil, "search query for greynoise (example: -greynoise 'query.txt')", goflags.FileStringSliceOptions),
9799
)
98100

99101
flagSet.CreateGroup("config", "Config",
@@ -169,7 +171,8 @@ func ParseOptions() *Options {
169171
len(options.Odin),
170172
len(options.BinaryEdge),
171173
len(options.Onyphe),
172-
len(options.Driftnet)) {
174+
len(options.Driftnet),
175+
len(options.GreyNoise)) {
173176
options.Engine = append(options.Engine, "shodan")
174177
}
175178

@@ -241,7 +244,8 @@ func (options *Options) validateOptions() error {
241244
len(options.Odin),
242245
len(options.BinaryEdge),
243246
len(options.Onyphe),
244-
len(options.Driftnet)) {
247+
len(options.Driftnet),
248+
len(options.GreyNoise)) {
245249
return errors.New("no query provided")
246250
}
247251

@@ -268,7 +272,8 @@ func (options *Options) validateOptions() error {
268272
len(options.Odin),
269273
len(options.BinaryEdge),
270274
len(options.Onyphe),
271-
len(options.Driftnet)) {
275+
len(options.Driftnet),
276+
len(options.GreyNoise)) {
272277
return errors.New("no engine specified")
273278
}
274279

@@ -312,6 +317,7 @@ func appendAllQueries(options *Options) {
312317
appendQuery(options, "binaryedge", options.BinaryEdge...)
313318
appendQuery(options, "onyphe", options.Onyphe...)
314319
appendQuery(options, "driftnet", options.Driftnet...)
320+
appendQuery(options, "greynoise", options.GreyNoise...)
315321
}
316322

317323
func (options *Options) useAwesomeSearchQueries(awesomeSearchQueries []string) error {

0 commit comments

Comments
 (0)