Java爬虫 | HttpClient的用法

Java爬虫 | HttpClient的用法

HttpClient主要用来抓取数据

用HttpClient发送get请求

public class UrlPool {
    public static void main(String[] args) throws IOException {
        //1.打开浏览器，创建HttpClients对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //2.输入网址，发起GET请求，创建
        HttpGet httpGet = new HttpGet("http://www.itcast.cn/");
        System.out.println("请求：" + httpGet);

        //3.使用httpClient对象发起请求，返回响应
        CloseableHttpResponse response = httpClient.execute(httpGet);

        //4.解析响应，获取数据
        if (response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity = response.getEntity();//响应体
            String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content.length());
        }

        response.close();
        httpClient.close();
    }
}

 用HttpClient发送带参数get请求

public class UrlPool {
    public static void main(String[] args) throws Exception {
        //打开浏览器，创建HttpClients对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //设置请求地址：http://yun.itheima.com/search?keys=Java
        //创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com/search");
        //设置参数
        uriBuilder.setParameter("keys","Java");

        //创建HttpGet对象，设置url访问地址
        HttpGet httpGet = new HttpGet(uriBuilder.build());

        //使用httpClient对象发起请求，返回响应
        CloseableHttpResponse response = httpClient.execute(httpGet);

        //解析响应，获取数据
        if (response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity = response.getEntity();//响应体
            String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content);
        }

        response.close();
        httpClient.close();
    }
}

 用HttpClient发送post请求(和get区别不大)

public class UrlPool {
    public static void main(String[] args) throws IOException {
        //1.打开浏览器，创建HttpClients对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //2.输入网址，发起GET请求，创建
        HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
        System.out.println("请求：" + httpPost);

        //3.使用httpClient对象发起请求，返回响应
        CloseableHttpResponse response = httpClient.execute(httpPost);

        //4.解析响应，获取数据
        if (response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity = response.getEntity();//响应体
            String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content.length());
        }

        response.close();
        httpClient.close();
    }
}

用HttpClient发送带参数post请求 

public class UrlPool {
    public static void main(String[] args) throws IOException {
        //打开浏览器，创建HttpClients对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //输入网址，发起GET请求，创建
        HttpPost httpPost = new HttpPost("http://yun.itheima.com/search");
        System.out.println("请求：" + httpPost);

        //声明List集合，封装表单中的参数
        List params = new ArrayList();

        //设置请求地址：http://yun.itheima.com/search?keys=Java
        params.add(new BasicNamevaluePair("keys","Java"));

        //创建表单Entity对象，第一个参数就是封装好的表单数据，第二个参数就是编码
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params,"utf8");

        //设置表单中的Entity对象到Post请求中
        httpPost.setEntity(formEntity);

        //使用httpClient对象发起请求，返回响应
        CloseableHttpResponse response = httpClient.execute(httpPost);

        //解析响应，获取数据
        if (response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity = response.getEntity();//响应体
            String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content);
        }

        response.close();
        httpClient.close();
    }
}

使用连接池管理 

public class UrlPool {
    public static void main(String[] args) throws IOException {
        //创建连接池管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        //设置最大连接数
        cm.setMaxTotal(100);
        //设置每个主机（目标网址）最大连接数
        cm.setDefaultMaxPerRoute(10);
        //使用连接池管理器发起请求
        doDet(cm);
        doDet(cm);
    }

    private static void doDet(PoolingHttpClientConnectionManager cm) throws IOException {
        //不用每次都创建新的连接，从连接池中获取即可
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        //2.输入网址，发起GET请求，创建
        HttpGet httpGet = new HttpGet("http://www.itcast.cn/");
        System.out.println("请求：" + httpGet);

        //3.使用httpClient对象发起请求，返回响应
        CloseableHttpResponse response = httpClient.execute(httpGet);

        //4.解析响应，获取数据
        if (response.getStatusLine().getStatusCode() == 200){
            HttpEntity httpEntity = response.getEntity();//响应体
            String content = EntityUtils.toString(httpEntity,"utf8");

            System.out.println(content.length());
        }

        response.close();
        //不能关闭httpClient，有连接池管理
//        httpClient.close();
    }
}

配置请求信息