关注它，不迷路。
本文章中所有内容仅供学习交流，不可用于任何商业用途和非法用途，否则后果自负，如有侵权，请联系作者立即删除！
一.题目地址
https://match.yuanrenxue.cn/match/13二.抓包分析
先打开控制台，然后再打开上面的网站，查看数据所在的页面:
发现每次请求数据页，都会先请求下面这个url：
https://match.yuanrenxue.cn/api2/13才看数据，发现是一段js代码:
控制台运行:
原来是设置了cookie，那就很简单了。直接交给AI吧。
三.编写Python代码
import jsonimport refrom typing import Tuple import requests BASE_URL = "https://match.yuanrenxue.cn"MATCH_URL = f"{BASE_URL}/match/13"API2_URL = f"{BASE_URL}/api2/13"QUESTION_URL = f"{BASE_URL}/api/question/13"SESSIONID = "XXXXXX"USER_AGENT = "yuanrenxue" FALSE_TEXT = "false"TRUE_TEXT = "true"OBJECT_TEXT = "[object Object]" def _replace_indexed_token(expression: str, pattern: str, alphabet: str) -> str: def repl(match: re.Match[str]) -> str: index = int(match.group(1)) try: char = alphabet[index] except IndexError as exc: raise ValueError(f"Unexpected token index {index} for {alphabet!r}") from exc return f"('{char}')" return re.sub(pattern, repl, expression) def decode_cookie_assignment(js_code: str) -> Tuple[str, str]: script = js_code.strip() prefix = "document.cookie=" if not script.startswith(prefix): raise ValueError(f"Unexpected /api2/13 response: {script[:80]!r}") expression = script[len(prefix) :] if expression.endswith(";"): expression = expression[:-1] expression = expression.replace('\\"', '"') expression = re.sub(r"\s+", "", expression) expression = _replace_indexed_token( expression, r"\(\(\[\]\+!\[\]\)\[(\d+)\]\)", FALSE_TEXT, ) expression = _replace_indexed_token( expression, r"\(\(\[\]\+\!\!\[\]\)\[(\d+)\]\)", TRUE_TEXT, ) expression = _replace_indexed_token( expression, r'\(\(\{\}\+""\)\[(\d+)\]\)', OBJECT_TEXT, ) pieces = re.findall(r"\('([^']*)'\)", expression) if not pieces: raise ValueError(f"Unable to decode cookie expression: {expression[:120]!r}") assignment = "".join(pieces) if "=" not in assignment: raise ValueError(f"Decoded assignment is invalid: {assignment!r}") name, remainder = assignment.split("=", 1) value = remainder.split(";", 1)[0] return name, value def build_session() -> requests.Session: session = requests.Session() session.cookies.set("sessionid", SESSIONID, domain="match.yuanrenxue.cn", path="/") session.headers.update( { "User-Agent": USER_AGENT, "Referer": MATCH_URL, "X-Requested-With": "XMLHttpRequest", } ) return session def fetch_page(session: requests.Session, page: int) -> list[int]: api2_response = session.get( API2_URL, headers={"Accept": "*/*"}, timeout=10, ) api2_response.raise_for_status() cookie_name, cookie_value = decode_cookie_assignment(api2_response.text) session.cookies.set(cookie_name, cookie_value, domain="match.yuanrenxue.cn", path="/") print (cookie_name,cookie_value) question_response = session.get( QUESTION_URL, params={"page": page, "pageSize": 10, "kw": ""}, headers={"Accept": "application/json, text/javascript, */*; q=0.01"}, timeout=10, ) question_response.raise_for_status() payload = question_response.json() if "data" not in payload or not isinstance(payload["data"], list): raise ValueError(f"Unexpected question payload: {payload!r}") return payload["data"] def main() -> None: session = build_session() all_numbers: list[int] = [] page_results: dict[int, list[int]] = {} for page in range(1, 6): numbers = fetch_page(session, page) page_results[page] = numbers all_numbers.extend(numbers) print(f"page {page}: {json.dumps(numbers, ensure_ascii=False)}") print(f"sum: {sum(all_numbers)}") if __name__ == "__main__": main()运行结果:
今天的分享就到这里，感谢阅读。
欢迎加入知识星球，学习更多AST和爬虫技巧。
