Skip to content

Examples with httpcloak

Each example below runs a whole solve. It fetches the page, decides what it is (see Challenges and blocks), runs the loop, then fetches the page again with cf_clearance. Every request to the site goes through httpcloak 1.7.3 or later, which gives your connections Chrome’s TLS and HTTP/2 fingerprint.

Replace YOUR_API_KEY and the proxy, and point PAGE at the site.

import base64
import time
from urllib.parse import urlparse
import httpcloak # pip install "httpcloak>=1.7.3"
import requests
API = "https://cf.antibotapi.com"
API_KEY = "YOUR_API_KEY"
PAGE = "https://www.example.com/"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36"
LANGUAGE = "en-GB,en-US;q=0.9,en;q=0.8"
session = httpcloak.Session(
preset="chrome-155-windows",
proxy="http://user:pass@host:port",
http_version="h2",
allow_redirects=False,
)
def api(path, body):
r = requests.post(API + path, json=body, headers={"x-api-key": API_KEY}, timeout=30)
if r.status_code != 200:
raise RuntimeError(f"{path}: {r.status_code} {r.text}")
return r.json()
def flat(headers):
"""httpcloak's headers ({name: [values]}) as /next takes them: lower-case names, repeated values joined."""
return {name.lower(): ", ".join(values) for name, values in headers.items()}
def cookie_header(url):
"""The jar's cookies for an address, as Chrome's Cookie header lists them."""
u = urlparse(url)
path = u.path or "/"
sent = []
for c in session.get_cookies():
domain = c.domain.lstrip(".")
if (u.hostname == domain or u.hostname.endswith("." + domain)) and path.startswith(c.path or "/") \
and (u.scheme == "https" or not c.secure):
sent.append(c)
sent.sort(key=lambda c: -len(c.path or "/"))
return "; ".join(f"{c.name}={c.value}" for c in sent)
def send(req):
"""One request we were given, sent as given; what came back, as /next takes it."""
time.sleep(req["wait_ms"] / 1000)
headers = []
for name, value in req["headers"]:
if name == "cookie":
value = cookie_header(req["url"])
if not value:
continue
headers.append((name, value))
started = time.monotonic()
try:
r = session.request(req["method"], req["url"], data=req.get("body"), exact_headers=headers,
timeout=req["timeout_ms"] / 1000 or None)
except httpcloak.HTTPCloakError:
if req["timeout_ms"] and time.monotonic() - started >= req["timeout_ms"] / 1000:
return {"status": 0, "headers": {}, "body": ""}
raise
return {
"status": r.status_code,
"headers": flat(r.headers),
"body": base64.b64encode(r.content).decode(),
}
def solve(url, html):
turn = api("/solves", {"url": url, "html": html, "user_agent": UA})
solve_id, req = turn["id"], turn["request"]
while True:
turn = api(f"/solves/{solve_id}/next", send(req))
if turn.get("done"):
return turn
req = turn["request"]
r = session.get(PAGE, headers={"accept-language": LANGUAGE})
if flat(r.headers).get("cf-mitigated") == "challenge":
result = solve(PAGE, r.text)
if not result["cleared"]:
raise RuntimeError("Cloudflare challenged again: check the client and the proxy")
r = session.get(PAGE, headers={"accept-language": LANGUAGE})
print(r.status_code, "with cf_clearance")
elif "challenges.cloudflare.com/turnstile/v0/api.js" in r.text:
token = solve(PAGE, r.text)["token"]
print("cf-turnstile-response:", token) # post it with the page's form
elif r.status_code in (403, 429):
raise RuntimeError("Blocked: not a challenge, see the error code on the page")
else:
print(r.status_code, "no challenge")