diff --git a/README.md b/README.md index 7789558..b7bb612 100644 --- a/README.md +++ b/README.md @@ -279,24 +279,25 @@ This works like `request.get`, with the addition of the postData parameter. Note ## Environment variables -| Name | Default | Notes | -| ------------------ | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. | -| LOG_FILE | none | Path to capture log to file. Example: `/config/flaresolverr.log`. | -| LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. | -| PROXY_URL | none | URL for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `http://127.0.0.1:8080`. | -| PROXY_USERNAME | none | Username for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `testuser`. | -| PROXY_PASSWORD | none | Password for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `testpass`. | -| CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. | -| TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. | -| LANG | none | Language used in the web browser. Example: `LANG=en_GB`. | -| HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. | -| DISABLE_MEDIA | false | To disable loading images, CSS, and other media in the web browser to save network bandwidth. | -| TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. | -| PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. | -| HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. | -| PROMETHEUS_ENABLED | false | Enable Prometheus exporter. See the Prometheus section below. | -| PROMETHEUS_PORT | 8192 | Listening port for Prometheus exporter. See the Prometheus section below. | +| Name | Default | Notes | +|----------------------| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. | +| LOG_FILE | none | Path to capture log to file. Example: `/config/flaresolverr.log`. | +| LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. | +| PROXY_URL | none | URL for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `http://127.0.0.1:8080`. | +| PROXY_USERNAME | none | Username for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `testuser`. | +| PROXY_PASSWORD | none | Password for proxy. Will be overwritten by `request` or `sessions` proxy, if used. Example: `testpass`. | +| CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. | +| TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. | +| LANG | none | Language used in the web browser. Example: `LANG=en_GB`. | +| HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. | +| DISABLE_MEDIA | false | To disable loading images, CSS, and other media in the web browser to save network bandwidth. | +| BROWSER_WAIT_TIMEOUT | 1 | Seconds to wait for the web browser to reach an expected page state on each attempt. Increase it on slow hosts or slow websites. | +| TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. | +| PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. | +| HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. | +| PROMETHEUS_ENABLED | false | Enable Prometheus exporter. See the Prometheus section below. | +| PROMETHEUS_PORT | 8192 | Listening port for Prometheus exporter. See the Prometheus section below. | Environment variables are set differently depending on the operating system. Some examples: diff --git a/src/flaresolverr.py b/src/flaresolverr.py index da99ff5..fedba8c 100644 --- a/src/flaresolverr.py +++ b/src/flaresolverr.py @@ -90,6 +90,7 @@ if __name__ == "__main__": log_file = os.environ.get('LOG_FILE', None) log_html = utils.get_config_log_html() headless = utils.get_config_headless() + browser_wait_timeout = utils.get_config_browser_wait_timeout() server_host = os.environ.get('HOST', '0.0.0.0') server_port = int(os.environ.get('PORT', 8191)) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 814aeff..a74d08f 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -53,7 +53,6 @@ TURNSTILE_SELECTORS = [ "input[name='cf-turnstile-response']" ] -SHORT_TIMEOUT = 1 SESSIONS_STORAGE = SessionsStorage() @@ -422,6 +421,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge logging.info("Challenge detected. Selector found: " + selector) break + browser_wait_timeout = utils.get_config_browser_wait_timeout() attempt = 0 if challenge_found: while True: @@ -430,12 +430,12 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge # wait until the title changes for title in CHALLENGE_TITLES: logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title) - WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title)) + WebDriverWait(driver, browser_wait_timeout).until_not(title_is(title)) # then wait until all the selectors disappear for selector in CHALLENGE_SELECTORS: logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector) - WebDriverWait(driver, SHORT_TIMEOUT).until_not( + WebDriverWait(driver, browser_wait_timeout).until_not( presence_of_element_located((By.CSS_SELECTOR, selector))) # all elements not found @@ -453,7 +453,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge logging.debug("Waiting for redirect") # noinspection PyBroadException try: - WebDriverWait(driver, SHORT_TIMEOUT).until(staleness_of(html_element)) + WebDriverWait(driver, browser_wait_timeout).until(staleness_of(html_element)) except Exception: logging.debug("Timeout waiting for redirect") diff --git a/src/utils.py b/src/utils.py index 9edbe28..5e12178 100644 --- a/src/utils.py +++ b/src/utils.py @@ -32,6 +32,10 @@ def get_config_disable_media() -> bool: return os.environ.get('DISABLE_MEDIA', 'false').lower() == 'true' +def get_config_browser_wait_timeout() -> int: + return int(os.environ.get('BROWSER_WAIT_TIMEOUT', 1)) + + def get_flaresolverr_version() -> str: global FLARESOLVERR_VERSION if FLARESOLVERR_VERSION is not None: