selenium获取不到标签内容

Get the visible (i.e. not hidden by CSS) text of this element, including sub-elements.

WebElement.getText();

该方法只能获取到“可见的”标签的内容,可使用WebElement.getAttribute(“innerText”); 获取隐藏标签的内容;

部分异步加载的网页,可能需要滚动条滚动到指定位置才加载,可设置浏览器窗口高度使网页的代码判断浏览器可见高度达到了加载条件,触发后续页面加载;

WebDriver.manage().window().setSize(new Dimension(1920, 1080 * 2));

eclipse maven 项目添加 lombok 支持

Step 1: 为项目添加依赖

在pom.xml中添加依赖

<dependency>
    <groupId>org.projectlombok</groupId>
    <artifactId>lombok</artifactId>
</dependency>

等待eclipse下载完依赖包

Step 2: 为IDE添加 lombok 支持

在CMD控制台进去本地Maven仓库目录,找到 lombok 的 jar 文件,执行命令
在弹窗中选择需要安装lombok支持的IDE

Step 3: 安装完成后重启IDE

现在就可以在项目中使用 lombok 的注解了。

spring-cloud-gateway-example

package com.example.demo;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.cloud.gateway.filter.ratelimit.KeyResolver;
import org.springframework.cloud.gateway.filter.ratelimit.RateLimiter;
import org.springframework.cloud.gateway.filter.ratelimit.RedisRateLimiter;
import org.springframework.cloud.gateway.route.RouteLocator;
import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder;
import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder.Builder;
import org.springframework.context.annotation.Bean;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpHeaders;
import org.springframework.http.server.reactive.ServerHttpRequest;
import org.springframework.http.server.reactive.ServerHttpResponse;
import org.springframework.util.CollectionUtils;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;

import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

@SpringBootApplication
public class GatewayApplication {

	private static final Logger LOGGER = LoggerFactory.getLogger(GatewayApplication.class);
	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();

	@Autowired
	private WhiteList whiteList;

	@Bean
	@Order(0)
	GlobalFilter globalFilter0() {
		return (exchange, chain) -> {
			ServerHttpRequest request = exchange.getRequest();
			ServerHttpResponse response = exchange.getResponse();
			List token = request.getHeaders().get("token");
			if (whiteList.getWhiteList().contains(request.getURI().getPath())) {
				return chain.filter(exchange);
			} else if (!CollectionUtils.isEmpty(token) && token.get(0).equals("abc")) {
				LOGGER.info("Token: " + token.get(0));
				request = exchange.getRequest().mutate().header("userId", "0").headers(c -> c.remove("token")).build();
				return chain.filter(exchange.mutate().request(request).build());
			} else {
				response.getHeaders().add("Content-Type", "application/json;charset=utf-8");
				Map res = new HashMap<>();
				res.put("code", 403);
				res.put("data", "");
				res.put("message", "invalid token");
				try {
					return response.writeWith(Flux.just(response.bufferFactory().wrap(OBJECT_MAPPER.writeValueAsBytes(res))));
				} catch (JsonProcessingException e) {
					e.printStackTrace();
				}
				return null;
			}
		};
	}

	@Bean
	@Order(1)
	GlobalFilter globalFilter1() {
		return (exchange, chain) -> {
			LOGGER.info(System.currentTimeMillis() + "");
			return chain.filter(exchange).then(Mono.fromRunnable(() -> {
				HttpHeaders headers = exchange.getResponse().getHeaders();
				headers.add("Access-Control-Allow-Origin", "*");
				headers.add("Access-Control-Allow-Credentials", "true");
				LOGGER.info(System.currentTimeMillis() + "");
			}));
		};
	}

	@Bean
	KeyResolver keyResolver() {
		return (exchange) -> {
			return Mono.just(exchange.getRequest().getRemoteAddress().getAddress().getHostAddress());
		};
	}

	@Bean
	RateLimiter< ?> rateLimiter() {
		return new RedisRateLimiter(4, 16);
	}

	@Bean
	RouteLocator customRouteLocator(RouteLocatorBuilder builder) {
		Builder b = builder.routes();
		// @formatter:off
		b.route("h5", r -> r.path("/h5/**")
				.filters(f -> f.requestRateLimiter(c -> c.setRateLimiter(rateLimiter()).setKeyResolver(keyResolver())))
				.uri("http://192.168.100.252:801"));
		b.route("otc", r -> r.path("/otc/**")
				.filters(f -> f.requestRateLimiter(c -> c.setRateLimiter(rateLimiter()).setKeyResolver(keyResolver())))
				.uri("http://192.168.100.252:801"));
		// @formatter:on
		return b.build();
	}

	public static void main(String[] args) {
		SpringApplication.run(GatewayApplication.class, args);
	}

}

package com.example.demo;

import java.util.List;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

@Component
@ConfigurationProperties(prefix = "bitx")
public class WhiteList {

	private List whiteList;

	public List getWhiteList() {
		return whiteList;
	}

	public void setWhiteList(List whiteList) {
		this.whiteList = whiteList;
	}

}

MySQL 数据导入 Elasticsearch

JAVA:

public class JDBCImporter {

	private static final String TABLE_NAME = "table_name";
	private static final String URL = "JDBC连接字符串";
	private static final CloseableHttpClient HTTP_CLIENT = HttpClientBuilder.create().build();
	private static final StringBuffer STRING_BUFFER = new StringBuffer();
	private static final Gson GSON = new GsonBuilder().create();

	public static void main(String[] args) throws Exception {
		Connection connection = DriverManager.getConnection(URL);
		com.mysql.cj.core.result.Field[] fields = getFields(connection, TABLE_NAME);

		String sql = "SELECT * FROM " + TABLE_NAME + " WHERE id > ? ORDER BY id ASC LIMIT 10000"; //每次批量索引10000条

		int bulkSize = 0; //已索引的总数量
		long lastId = 0;  //已索引的最后一个ID
		do {
			PreparedStatement ps = connection.prepareStatement(sql);
			ps.setLong(1, lastId);
			ResultSet rs = ps.executeQuery();
			while (rs.next()) {
				bulkSize++;
				lastId = rs.getLong("id");
				Map row = new HashMap<>();
				for (int i = 0; i < fields.length; i++) {
					String key = fields[i].getColumnLabel();
					row.put(key, rs.getObject(key));
				}
				STRING_BUFFER.append("{\"index\":{\"_id\":\"" + lastId + "\"}}\n").append(GSON.toJson(row)).append("\n");
			}
			StringEntity entity = new StringEntity(STRING_BUFFER.toString(), ContentType.APPLICATION_JSON);
			HttpPost post = new HttpPost("http://192.168.56.200:9200/" + TABLE_NAME + "/" + TABLE_NAME + "/_bulk");
			post.setEntity(entity);
			CloseableHttpResponse response = HTTP_CLIENT.execute(post);
			System.out.println(EntityUtils.toString(response.getEntity()));
			STRING_BUFFER.delete(0, STRING_BUFFER.length());
			rs.close();
			ps.close();
		} while (bulkSize % 10000 == 0);

		connection.close();
	}

	// 读取MYSQL数据表的字段信息
	private static com.mysql.cj.core.result.Field[] getFields(Connection connection, String tableName) throws Exception {
		ResultSet rs = connection.prepareStatement("SELECT * FROM " + tableName + " WHERE 1=0").executeQuery();
		ResultSetMetaData metaData = rs.getMetaData();
		Field field = rs.getMetaData().getClass().getDeclaredField("fields");
		field.setAccessible(true);
		com.mysql.cj.core.result.Field[] fields = (com.mysql.cj.core.result.Field[]) field.get(metaData);
		rs.close();
		return fields;
	}

}

PHP:

public function main(){
	ES::$INDEX = 'log_apprequestlogs';
	// 取出已索引记录的最大ID
	$maxId = ES::bodySearch('log_apprequestlogs', json_decode('{"aggs":{"max_id":{"max":{"field":"id"}}}}',true));
	$start = $maxId['aggregations']['max_id']['value'];
	do{
		$i = 0;
		$Table = TableRegistry::get('LogApprequestlogs');
		$result = $Table->find('all',[
			'conditions' => "id > {$start}", //过滤已索引的记录
			'order' => 'id asc',
			'limit' => 5000 //每次批量处理5000条
		]);
		$postBodyArray = [];
		try{
			foreach($result as $row){
				$i++;
				$start = $row->id;
				$postBodyArray[] = json_encode(['index'=>['_id'=>$row->id]]);
				$postBodyArray[] = json_encode($row, JSON_UNESCAPED_UNICODE);
			}
		}catch(\Exception $e){
			echo $e->getMessage();
			sleep(10);
			continue;
		}
		try{
			ES::bulk('log_apprequestlogs', implode("\n", $postBodyArray)); // 使用 bulk API 批量索引
		}catch(\Exception $e){
			echo $e->getMessage();
			sleep(10);
			continue;
		}
		if($i%5000 != 0 || $i == 0){
			sleep(10); // 没有新数据时,等待10秒再继续处理。
			$i = 5000;
		}
	}while($i%5000 == 0);
}

Apache Mina 断包、粘包

/**
 * 解码类继承 CumulativeProtocolDecoder 类
 */
public class Decoder extends CumulativeProtocolDecoder {
    ......
    @Override
    protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception {
        //读取当前 IoBuffer 中已接收到的数据,判断是否得到了完整的数据包
        if( 得到了可以解析的完整的数据包 ){
            ......
            return true; //通知父类的 decode 方法,当前接收并读取的数据已处理
        }else{
            in.rewind();
            return false; //通知父类的 decode 方法,当前数据不完整,放弃本次读取,下个数据包到来时再做处理
        }
    }
    ......
}

试用 WebSocket

public class MyWebSocketServer extends WebSocketServer {

	private static MyWebSocketServer server;

	private static InetSocketAddress address = new InetSocketAddress(8080);

	public MyWebSocketServer() throws UnknownHostException {
		super(address);
	}

	@Override
	public void onOpen(WebSocket conn, ClientHandshake handshake) {
		System.out.println("open");
	}

	@Override
	public void onClose(WebSocket conn, int code, String reason, boolean remote) {
		System.out.println("close");
	}

	@Override
	public void onMessage(WebSocket conn, String message) {
		System.out.println(message);
		if (message.equals("close")) {
			try {
				server.stop(3000);
			} catch (IOException e) {
				e.printStackTrace();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		} else {
			conn.send(message);
		}
	}

	@Override
	public void onError(WebSocket conn, Exception ex) {
		System.out.println("error");
	}

	public static void main(String[] args) {
		try {
			server = new MyWebSocketServer();
		} catch (UnknownHostException e1) {
			e1.printStackTrace();
		}
		server.start();
	}

}
<input id="t" type="text" /><input id="b" type="button" value="Send" />
<script>
var wsServer = 'ws://192.168.56.1:8080';
var websocket = new WebSocket(wsServer);
websocket.onopen = function(evt){
	console.log("open");
	document.getElementById("b").addEventListener("click", function(evt){
		websocket.send(document.getElementById("t").value);
	}, false);
};
websocket.onclose = function(evt){
	console.log("close");
};
websocket.onmessage = function(evt){
	console.log(evt.data + " at " + evt.timeStamp);
};
websocket.onerror = function(evt){
	console.log("error");
};
</script>

java URLConnection 实现文件上传

* 此工具类不建议上传大文件(百兆以上),大文件有内存溢出的风险。如果需要传大文件,可使用 hc 另行实现。
* 此工具类适用于普通网站中用户上传的图片、小附件等文件存储。

	public static HashMap upload(MultipartFile file, String domain, boolean isProtected){
		HashMap map = null;
		
		URI uri = URI.create("http://static.myechinese.com/up");
		InetSocketAddress sa = new InetSocketAddress("218.106.246.55", 80);
		Proxy proxy = new Proxy(Type.HTTP, sa);
		try {
			URLConnection conn = uri.toURL().openConnection(proxy);
			String boundary = "---------------------------" + System.currentTimeMillis();
			String boundaryInContent = "--" + boundary;
			String rn = "\r\n";
			conn.addRequestProperty("Connection", "keep-alive");
			conn.addRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
			conn.setDoInput(true);
			conn.setDoOutput(true);
			conn.setConnectTimeout(1000);
			conn.setUseCaches(false);
			conn.connect();
			OutputStream out = conn.getOutputStream();
			StringBuilder sb = new StringBuilder();
			sb.append(boundaryInContent).append(rn);
			sb.append("Content-Disposition: form-data; name=domain").append(rn).append(rn);
			sb.append(domain).append(rn);
			sb.append(boundaryInContent).append(rn);
			sb.append("Content-Disposition: form-data; name=isProtected").append(rn).append(rn);
			sb.append(isProtected).append(rn);
			sb.append(boundaryInContent).append(rn);
			sb.append("Content-Disposition: form-data; name=file; filename=" + file.getOriginalFilename()).append(rn);
			sb.append("Content-Type: " + file.getContentType()).append(rn).append(rn);
			out.write(sb.toString().getBytes());
			out.write(file.getBytes());
			sb.delete(0, sb.length());
			sb.append(rn).append(boundaryInContent).append("--").append(rn).append(rn);
			out.write(sb.toString().getBytes());
			out.flush();
			out.close();
			BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
			String res = reader.readLine();
			reader.close();
			System.out.println("Uploader: " + res);
			map = new Gson().fromJson(res, HashMap.class);// Gson 有点坑,数字默认给转成 Double 类型
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		return map;
	}

java 后台服务无故退出

服务器上跑着一个小工具,总是在一段时间后无缘无故的退出,程序日志没有任何异常输出。
后来在系统日志 /var/log/messages 文件中发现了两条相关信息:

Apr  6 19:05:46 localhost kernel: [20477]   501 20477    89051     4497   0       0             0 php-fpm
Apr  6 19:05:46 localhost kernel: [20557]     0 20557   537756    49736   0       0             0 java
Apr  6 19:05:46 localhost kernel: Out of memory: Kill process 20557 (java) score 367 or sacrifice child
Apr  6 19:05:46 localhost kernel: Killed process 20557, UID 0, (java) total-vm:2151024kB, anon-rss:198936kB, file-rss:8kB

将JAVA程序添加至系统服务

#!/bin/sh

### BEGIN INIT INFO
# Provides: java service
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Start up the java service daemon.
# Description:       Start up the java service daemon.
### END INIT INFO

BASE_DIR="/home/yangxianchao/pq"
PID=$(ps -ef | grep com/st/pq/SendComments | awk '{if($3==1){print $2}}')

cd $BASE_DIR

start(){
  if [ "$PID" == "" ]
  then
    for i in *.jar
    do
      CLASS_PATH=$CLASS_PATH:$i
    done
    java -cp $CLASS_PATH com/st/pq/SendComments > /dev/null 2>&1 &
    sleep 1
    echo "Start."
  else
    echo "It's running."
  fi
}
stop(){
  if [ "$PID" == "" ]
  then
    echo "It's not running."
  else
    kill $PID
    sleep 1
    echo "Stop."
  fi
}
status(){
  if [ "$PID" == "" ]
  then
    echo "stop"
  else
    echo "running"
  fi
}
case $1 in
  start)
    start
  ;;
  stop)
    stop
  ;;
  status)
    status
  ;;
  *)
    echo "start|stop|status"
  ;;
esac