crawl-bid/src/main/java/me/yoqi/crawlbid/App.java

package me.yoqi.crawlbid;

import java.awt.Image;
import java.util.function.Consumer;
import java.util.regex.Pattern;

import org.openqa.selenium.OutputType;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.devtools.DevTools;
import org.openqa.selenium.devtools.network.Network;
import org.openqa.selenium.devtools.network.model.ResourceType;
import org.openqa.selenium.devtools.network.model.ResponseReceived;

import me.yoqi.crawlbid.model.Config;

/**
 * Hello world!
 *
 */
public class App {
	static DevTools devTools;
	static ChromeDriver driver;

	public static void main(String[] args) {

		String chromeDriverPath = Config.getString("App.chromeDriverPath");
		System.setProperty("webdriver.chrome.driver", chromeDriverPath);
		ChromeOptions options = new ChromeOptions();
		options.addArguments("lang=zh_CN.UTF-8",
				"user-agent=\"Mozilla/5.0 (iPod; U; CPU iPhone OS 2_1 like Mac OS X; ja-jp) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5F137 Safari/525.20\"",
				"cookies=");
		// options.setProxy(proxy);

		driver = new ChromeDriver(options);

		// 打开devtools
		devTools = driver.getDevTools();
		devTools.createSession();
		devTools.send(
				Network.enable(java.util.Optional.empty(), java.util.Optional.empty(), java.util.Optional.empty()));

		String startUrl = Config.getString("App.startUrl");
		driver.get(startUrl);

		String pattern = Config.getString("App.patten");
		final StringBuilder sb = new StringBuilder();
		InterceptCallback<String> callback = new InterceptCallback<String>() {
			public void emit(String res) {
				sb.append(res);
			}
		};
		interceptResponse4XHR(pattern, callback);
		// 点击每页100条
		driver.findElementByCssSelector("#app div.el-input.el-input--mini.el-input--suffix").click();
		try {
			Thread.sleep(500);
		} catch (InterruptedException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		driver.findElementByXPath("/html/body/div[2]/div[1]/div[1]/ul/li[6]").click();
		System.out.println("res:" + sb);

		for (int i = 2; i < 4; i++) {
			// 点击下一页
			driver.findElementByCssSelector("#app button.btn-next").click();
			final StringBuilder sb1 = new StringBuilder();
			InterceptCallback<String> callback1 = new InterceptCallback<String>() {
				public void emit(String res) {
					sb1.append(res);
				}
			};

			try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			devTools.clearListeners();
			interceptResponse4XHR(pattern, callback1);
			// driver.close(); //关闭当前tab标签
			// devTools.close();
			System.out.println("res:" + sb1);
		}
		// driver.quit(); // 关闭浏览器
	}

	public static void interceptResponse4XHR(String pattern, InterceptCallback<String> callback) {
		// 请求监听
		devTools.addListener(Network.responseReceived(), new Consumer<ResponseReceived>() {
			public void accept(ResponseReceived responseReceived) {
				if (ResourceType.XHR != responseReceived.getType()) {
					return;
				}
				// 获取url,判断获取指定的ajax 接口请求
				boolean isMatch = Pattern.matches(pattern, responseReceived.getResponse().getUrl());
				if (isMatch) {
					String body = devTools.send(Network.getResponseBody(responseReceived.getRequestId())).getBody();
					callback.emit(body);
				}
			}
		});
	}

	public void screenShot() {
		// 截屏
		OutputType<Image> outputType = new OutputType<Image>() {

			@Override
			public Image convertFromBase64Png(String base64Png) {
				// TODO Auto-generated method stub
				return null;
			}

			@Override
			public Image convertFromPngBytes(byte[] png) {
				// TODO Auto-generated method stub
				return null;
			}
		};
		driver.getScreenshotAs(outputType);
	}
}