????日志解析，強推ELK！
????日志解析，強推ELK！
????日志解析，強推ELK！
（有時間我要搭一套 flag+1）
接上文，我這邊接了個BI項目，就是帆軟的BI項目，但是發(fā)現(xiàn)報表的錯誤日志里面實在是太多了，發(fā)現(xiàn)帆軟的決策報表里有個查詢的口子，于是寫了個接口調(diào)用工具拉取帆軟的報錯信息
搭建restTemplate連接池
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.10</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<dependency>
我用的boot版本是2.1.5
創(chuàng)建連接池
@Configuration
public class RestTemplateConfiguration {
/**
* 返回RestTemplate
* @param factory
* @return
*/
@Bean
public RestTemplate restTemplate(ClientHttpRequestFactory factory) {
RestTemplate restTemplate = new RestTemplate(factory);
return restTemplate;
}
/**
* ClientHttpRequestFactory接口的另一種實現(xiàn)方式（推薦使用），即：
* HttpComponentsClientHttpRequestFactory：底層使用Httpclient連接池的方式創(chuàng)建Http連接請求
* @return
*/
@Bean
public HttpComponentsClientHttpRequestFactory httpComponentsClientHttpRequestFactory() {
//Httpclient連接池，長連接保持30秒
PoolingHttpClientConnectionManager connectionManager =
new PoolingHttpClientConnectionManager(30, TimeUnit.SECONDS);
//設置總連接數(shù)
connectionManager.setMaxTotal(1000);
//設置同路由的并發(fā)數(shù)
connectionManager.setDefaultMaxPerRoute(1000);
//設置header
List<Header> headers = new ArrayList<Header>();
headers.add(new BasicHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04"));
headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
headers.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3"));
headers.add(new BasicHeader("Connection", "keep-alive"));
headers.add(new BasicHeader("RequestType", "queue"));
//創(chuàng)建HttpClient
HttpClient httpClient = HttpClientBuilder.create()
.setConnectionManager(connectionManager)
.setDefaultHeaders(headers)
.setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) //設置重試次數(shù)
.setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) //設置保持長連接
.build();
//創(chuàng)建HttpComponentsClientHttpRequestFactory實例
HttpComponentsClientHttpRequestFactory requestFactory =
new HttpComponentsClientHttpRequestFactory(httpClient);
//設置客戶端和服務端建立連接的超時時間
requestFactory.setConnectTimeout(20000);
//設置客戶端從服務端讀取數(shù)據(jù)的超時時間
requestFactory.setReadTimeout(20000);
//設置從連接池獲取連接的超時時間，不宜過長
requestFactory.setConnectionRequestTimeout(200);
//緩沖請求數(shù)據(jù)，默認為true。通過POST或者PUT大量發(fā)送數(shù)據(jù)時，建議將此更改為false，以免耗盡內(nèi)存
requestFactory.setBufferRequestBody(false);
return requestFactory;
}
}
不用連接池更快，反正也是個小工具
模擬登錄，抓起錯誤信息
public List<String> getCookie(){
HttpHeaders headers = new HttpHeaders();
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //map里面是請求體的內(nèi)容
map.add("fr_username", PASSWORD);
map.add("fr_password",PASSWORD);
headers.setContentType(MediaType.parseMediaType("application/x-www-form-urlencoded; charset=UTF-8"));
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(LOGIN_URL, request, String.class); //地址替換為自己的
return response.getHeaders().get("Set-Cookie");
}
拿到cookie信息后，獲取總量信息
/**
* 保存組Group
*/
@Test
public void getTotal(){
HttpHeaders headers = new HttpHeaders();
List<String> cookies = getCookie();
headers.put(HttpHeaders.COOKIE,cookies); //將cookie存入頭部
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //請求體給予內(nèi)容
map.add("pn", "1"); //應用名稱
map.add("pageCount", "10"); //執(zhí)行器名稱
map.add("startTime", START_DATE); //排序方式
map.add("endTime", END_DATE); //注冊方式 ： 0為
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(ERROR_INFO_URL, request, String.class);
if(!StringUtils.isEmpty(response.getBody())){
JSONObject jsonObject = JSONObject.parseObject(response.getBody());
int totalPage = Integer.parseInt(jsonObject.get("totalpages").toString());
List<ErrorEntity> errorEntityList = new ArrayList<>();
if(totalPage>1){
for(int i =1;i< totalPage;i++){
getErrorInfo(i,errorEntityList);
}
}else{
errorEntityList = JSONArray.parseArray(jsonObject.get("items").toString(),ErrorEntity.class);
}
EasyExcel.write(result +"180_FR_ERROR_" + START_DATE + ".xlsx", ErrorEntity.class).sheet(START_DATE).doWrite(errorEntityList);
}
}
獲取分頁信息后這里就是循環(huán)調(diào)用的過程了
/**
* 保存組Group
*/
public void getErrorInfo(Integer page,List<ErrorEntity> errorEntities){
HttpHeaders headers = new HttpHeaders();
List<String> cookies = getCookie();
headers.put(HttpHeaders.COOKIE,cookies); //將cookie存入頭部
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //請求體給予內(nèi)容
map.add("pn", page.toString()); //應用名稱
map.add("pageCount", "10"); //執(zhí)行器名稱
map.add("startTime", START_DATE); //排序方式
map.add("endTime", END_DATE); //注冊方式 ： 0為
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(ERROR_INFO_URL, request, String.class);
if(!StringUtils.isEmpty(response.getBody())){
JSONObject jsonObject = JSONObject.parseObject(response.getBody());
errorEntities.addAll(JSONArray.parseArray(jsonObject.get("items").toString(),ErrorEntity.class));
}
}
進行文件封裝
1576674727210.jpg
