package me.yoqi.crawlbid; import java.awt.Image; import java.util.function.Consumer; import java.util.regex.Pattern; import org.openqa.selenium.OutputType; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.devtools.DevTools; import org.openqa.selenium.devtools.network.Network; import org.openqa.selenium.devtools.network.model.ResourceType; import org.openqa.selenium.devtools.network.model.ResponseReceived; import me.yoqi.crawlbid.model.Config; /** * Hello world! * */ public class App { static DevTools devTools; static ChromeDriver driver; public static void main(String[] args) { String chromeDriverPath = Config.getString("App.chromeDriverPath"); System.setProperty("webdriver.chrome.driver", chromeDriverPath); ChromeOptions options = new ChromeOptions(); options.addArguments("lang=zh_CN.UTF-8", "user-agent=\"Mozilla/5.0 (iPod; U; CPU iPhone OS 2_1 like Mac OS X; ja-jp) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5F137 Safari/525.20\"", "cookies="); // options.setProxy(proxy); driver = new ChromeDriver(options); // 打开devtools devTools = driver.getDevTools(); devTools.createSession(); devTools.send( Network.enable(java.util.Optional.empty(), java.util.Optional.empty(), java.util.Optional.empty())); String startUrl = Config.getString("App.startUrl"); driver.get(startUrl); String pattern = Config.getString("App.patten"); final StringBuilder sb = new StringBuilder(); InterceptCallback callback = new InterceptCallback() { public void emit(String res) { sb.append(res); } }; interceptResponse4XHR(pattern, callback); // 点击每页100条 driver.findElementByCssSelector("#app div.el-input.el-input--mini.el-input--suffix").click(); try { Thread.sleep(500); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } driver.findElementByXPath("/html/body/div[2]/div[1]/div[1]/ul/li[6]").click(); System.out.println("res:" + sb); for (int i = 2; i < 4; i++) { // 点击下一页 driver.findElementByCssSelector("#app button.btn-next").click(); final StringBuilder sb1 = new StringBuilder(); InterceptCallback callback1 = new InterceptCallback() { public void emit(String res) { sb1.append(res); } }; try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } devTools.clearListeners(); interceptResponse4XHR(pattern, callback1); // driver.close(); //关闭当前tab标签 // devTools.close(); System.out.println("res:" + sb1); } // driver.quit(); // 关闭浏览器 } public static void interceptResponse4XHR(String pattern, InterceptCallback callback) { // 请求监听 devTools.addListener(Network.responseReceived(), new Consumer() { public void accept(ResponseReceived responseReceived) { if (ResourceType.XHR != responseReceived.getType()) { return; } // 获取url,判断获取指定的ajax 接口请求 boolean isMatch = Pattern.matches(pattern, responseReceived.getResponse().getUrl()); if (isMatch) { String body = devTools.send(Network.getResponseBody(responseReceived.getRequestId())).getBody(); callback.emit(body); } } }); } public void screenShot() { // 截屏 OutputType outputType = new OutputType() { @Override public Image convertFromBase64Png(String base64Png) { // TODO Auto-generated method stub return null; } @Override public Image convertFromPngBytes(byte[] png) { // TODO Auto-generated method stub return null; } }; driver.getScreenshotAs(outputType); } }