From b1cf7ae4554e47b64b49b7e1827d731ea71a4cc8 Mon Sep 17 00:00:00 2001 From: Joe Zhou <zhoushuqun@gmail.com> Date: Wed, 18 Dec 2024 21:05:25 +0800 Subject: [PATCH] Set page status code for SeleniumDownloader. Fixes #1185 --- .../us/codecraft/webmagic/downloader/PhantomJSDownloader.java | 3 ++- .../webmagic/downloader/selenium/SeleniumDownloader.java | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java index 31dfca75a..01f1af9a3 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java @@ -6,6 +6,7 @@ import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Task; import us.codecraft.webmagic.selector.PlainText; +import us.codecraft.webmagic.utils.HttpConstant; import java.io.*; @@ -96,7 +97,7 @@ public Page download(Request request, Task task) { page.setRawText(content); page.setUrl(new PlainText(request.getUrl())); page.setRequest(request); - page.setStatusCode(200); + page.setStatusCode(HttpConstant.StatusCode.CODE_200); } onSuccess(page, task); } catch (Exception e) { diff --git a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java index 874f8aef7..f6d2574fb 100644 --- a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java +++ b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java @@ -14,9 +14,11 @@ import us.codecraft.webmagic.downloader.AbstractDownloader; import us.codecraft.webmagic.selector.Html; import us.codecraft.webmagic.selector.PlainText; +import us.codecraft.webmagic.utils.HttpConstant; import java.io.Closeable; import java.io.IOException; +import java.net.http.HttpRequest; import java.util.Map; /** @@ -111,6 +113,7 @@ public Page download(Request request, Task task) { page.setHtml(new Html(content, request.getUrl())); page.setUrl(new PlainText(request.getUrl())); page.setRequest(request); + page.setStatusCode(HttpConstant.StatusCode.CODE_200); onSuccess(page, task); } catch (Exception e) { logger.warn("download page {} error", request.getUrl(), e);