Skip to content

feat(): find slider and move it to pass the human challange #430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
inpercima wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

feat(): find slider and move it to pass the human challange #430

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 53 additions & 4 deletions src/main/java/de/codefor/le/crawler/LvzPoliceTickerCrawler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -45,6 +46,14 @@ public class LvzPoliceTickerCrawler {

protected static final String LVZ_POLICE_TICKER_BASE_URL = LVZ_BASE_URL + "/themen/leipzig-polizei";

private static final long MIN_DELAY_MS = 1500;

private static final long MAX_ADDITIONAL_DELAY_MS = 3500;

private static final long MIN_DELAY_MS_2 = 200;

private static final long MAX_ADDITIONAL_DELAY_MS_2 = 500;

private final PoliceTickerRepository policeTickerRepository;

@Value("${spring.profiles.active:}")
Expand Down Expand Up @@ -93,6 +102,8 @@ private String initLoad(final String url) {
initWebDriver();
driver.get(url);

driver.switchTo().frame(0);

final var consentFrames = driver.findElements(By.cssSelector("iframe[id*=sp_message_iframe]"));
if (!consentFrames.isEmpty()) {
logger.debug("accept cookies first, it's an iframe");
Expand All @@ -116,7 +127,29 @@ private String initLoad(final String url) {

final var fusionAppElements = driver.findElements(By.id("fusion-app"));
if (fusionAppElements.isEmpty()) {
logger.warn("fusion-app element not found — site may be blocking the crawler (title: {})", driver.getTitle());
logger.warn("fusion-app element not found — site may be blocking the crawler (title: {})",
driver.getTitle());
final var slider = driver.findElement(By.cssSelector("div.slider"));
new Actions(driver)
.dragAndDropBy(slider, 80, 0)
.perform();

randomDelayMin();

new Actions(driver)
.dragAndDropBy(slider, 120, 0)
.perform();

randomDelay();

new Actions(driver)
.dragAndDropBy(slider, 100, 0)
.perform();

driver.switchTo().parentFrame();

randomDelay();

WebDriverScreenshot.take(driver, WebDriverScreenshot.REASON_BLOCKED);
throw new IllegalStateException("fusion-app element not found, page title: " + driver.getTitle());
}
Expand Down Expand Up @@ -156,9 +189,9 @@ private boolean loadMoreArticles() {
}

private void initWebDriver() {
driver = "dev".equals(activeProfile) || "prod".equals(activeProfile) ?
WebDriverManager.chromedriver().remoteAddress("http://chrome:4444/wd/hub").create() :
new ChromeDriver(new ChromeOptions()
driver = "dev".equals(activeProfile) || "prod".equals(activeProfile)
? WebDriverManager.chromedriver().remoteAddress("http://chrome:4444/wd/hub").create()
: new ChromeDriver(new ChromeOptions()
.addArguments("--headless")
.addArguments("--disable-blink-features=AutomationControlled")
.addArguments("--user-agent=" + USER_AGENT));
Expand Down Expand Up @@ -203,4 +236,20 @@ private static boolean shouldSkipUrl(final String detailLink) {
return false;
}

private void randomDelay() {
try {
Thread.sleep(MIN_DELAY_MS + ThreadLocalRandom.current().nextLong(MAX_ADDITIONAL_DELAY_MS));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}

private void randomDelayMin() {
try {
Thread.sleep(MIN_DELAY_MS_2 + ThreadLocalRandom.current().nextLong(MAX_ADDITIONAL_DELAY_MS_2));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}

}
Loading