Commit f475e3120003d2449faf3ad1242c20fd9e6d0344
0 parents
Exists in
master
create
Showing
20 changed files
with
838 additions
and
0 deletions
Show diff stats
1 | +++ a/pom.xml | |
... | ... | @@ -0,0 +1,108 @@ |
1 | +<?xml version="1.0" encoding="UTF-8"?> | |
2 | +<project xmlns="http://maven.apache.org/POM/4.0.0" | |
3 | + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |
4 | + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |
5 | + <modelVersion>4.0.0</modelVersion> | |
6 | + | |
7 | + <groupId>com.peony</groupId> | |
8 | + <artifactId>crawler-netty</artifactId> | |
9 | + <version>1.0-SNAPSHOT</version> | |
10 | + | |
11 | + <dependencies> | |
12 | + <dependency> | |
13 | + <groupId>io.netty</groupId> | |
14 | + <artifactId>netty-all</artifactId> | |
15 | + <version>4.0.25.Final</version> | |
16 | + </dependency> | |
17 | + | |
18 | + <dependency> | |
19 | + <groupId>junit</groupId> | |
20 | + <artifactId>junit</artifactId> | |
21 | + <version>4.12</version> | |
22 | + </dependency> | |
23 | + | |
24 | + <dependency> | |
25 | + <groupId>org.slf4j</groupId> | |
26 | + <artifactId>slf4j-api</artifactId> | |
27 | + <version>1.7.21</version> | |
28 | + </dependency> | |
29 | + | |
30 | + <dependency> | |
31 | + <groupId>org.slf4j</groupId> | |
32 | + <artifactId>log4j-over-slf4j</artifactId> | |
33 | + <version>1.7.21</version> | |
34 | + </dependency> | |
35 | + | |
36 | + <dependency> | |
37 | + <groupId>ch.qos.logback</groupId> | |
38 | + <artifactId>logback-core</artifactId> | |
39 | + <version>1.1.7</version> | |
40 | + </dependency> | |
41 | + | |
42 | + <dependency> | |
43 | + <groupId>ch.qos.logback</groupId> | |
44 | + <artifactId>logback-classic</artifactId> | |
45 | + <version>1.1.7</version> | |
46 | + </dependency> | |
47 | + | |
48 | + <dependency> | |
49 | + <groupId>com.peony.crawler</groupId> | |
50 | + <artifactId>crawler-util</artifactId> | |
51 | + <version>0.0.1-SNAPSHOT</version> | |
52 | + </dependency> | |
53 | + | |
54 | + <dependency> | |
55 | + <groupId>org.apache.hbase</groupId> | |
56 | + <artifactId>hbase-client</artifactId> | |
57 | + <version>1.2.2</version> | |
58 | + <exclusions> | |
59 | + <exclusion> | |
60 | + <groupId>org.slf4j</groupId> | |
61 | + <artifactId>slf4j-log4j12</artifactId> | |
62 | + </exclusion> | |
63 | + </exclusions> | |
64 | + </dependency> | |
65 | + | |
66 | + <dependency> | |
67 | + <groupId>com.qingcloud.qingstor</groupId> | |
68 | + <artifactId>qingstor</artifactId> | |
69 | + <version>1.0-SNAPSHOT</version> | |
70 | + </dependency> | |
71 | + </dependencies> | |
72 | + | |
73 | + <build> | |
74 | + <plugins> | |
75 | + <plugin> | |
76 | + <groupId>org.apache.maven.plugins</groupId> | |
77 | + <artifactId>maven-compiler-plugin</artifactId> | |
78 | + <configuration> | |
79 | + <source>1.7</source> | |
80 | + <target>1.7</target> | |
81 | + <encoding>UTF-8</encoding> | |
82 | + </configuration> | |
83 | + </plugin> | |
84 | + <plugin> | |
85 | + <artifactId>maven-assembly-plugin</artifactId> | |
86 | + <configuration> | |
87 | + <descriptorRefs> | |
88 | + <descriptorRef>jar-with-dependencies</descriptorRef> | |
89 | + </descriptorRefs> | |
90 | + <archive> | |
91 | + <manifest> | |
92 | + <mainClass>com.peony.netty.Application</mainClass> | |
93 | + </manifest> | |
94 | + </archive> | |
95 | + </configuration> | |
96 | + <executions> | |
97 | + <execution> | |
98 | + <id>make-assembly</id> | |
99 | + <phase>package</phase> | |
100 | + <goals> | |
101 | + <goal>single</goal> | |
102 | + </goals> | |
103 | + </execution> | |
104 | + </executions> | |
105 | + </plugin> | |
106 | + </plugins> | |
107 | + </build> | |
108 | +</project> | |
0 | 109 | \ No newline at end of file | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/Application.java | |
... | ... | @@ -0,0 +1,24 @@ |
1 | +package com.peony.netty; | |
2 | + | |
3 | +import com.peony.netty.controller.DocContentController; | |
4 | +import com.peony.netty.controller.DocHtmlController; | |
5 | +import com.peony.netty.http.Request; | |
6 | +import com.peony.netty.http.Response; | |
7 | +import com.peony.netty.web.Route; | |
8 | + | |
9 | +import static com.peony.netty.web.UriAdapter.get; | |
10 | + | |
11 | +public class Application { | |
12 | + public static void main(String[] args) throws Exception { | |
13 | + HttpServer server = new HttpServer(); | |
14 | + get("hello", new Route() { | |
15 | + @Override | |
16 | + public Object handle(Request request, Response response) { | |
17 | + return "Hello World 你好"; | |
18 | + } | |
19 | + }); | |
20 | + new DocContentController().routes(); | |
21 | + new DocHtmlController().routes(); | |
22 | + server.start(8000); | |
23 | + } | |
24 | +} | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/HttpServer.java | |
... | ... | @@ -0,0 +1,44 @@ |
1 | +package com.peony.netty; | |
2 | + | |
3 | +import com.peony.netty.handler.HttpServerHandler; | |
4 | +import io.netty.bootstrap.ServerBootstrap; | |
5 | +import io.netty.channel.ChannelFuture; | |
6 | +import io.netty.channel.ChannelInitializer; | |
7 | +import io.netty.channel.ChannelOption; | |
8 | +import io.netty.channel.EventLoopGroup; | |
9 | +import io.netty.channel.nio.NioEventLoopGroup; | |
10 | +import io.netty.channel.socket.SocketChannel; | |
11 | +import io.netty.channel.socket.nio.NioServerSocketChannel; | |
12 | +import io.netty.handler.codec.http.HttpRequestDecoder; | |
13 | +import io.netty.handler.codec.http.HttpResponseEncoder; | |
14 | + | |
15 | +public class HttpServer { | |
16 | + public void start(int port) throws Exception { | |
17 | + EventLoopGroup bossGroup = new NioEventLoopGroup(); | |
18 | + EventLoopGroup workerGroup = new NioEventLoopGroup(); | |
19 | + try { | |
20 | + ServerBootstrap b = new ServerBootstrap(); | |
21 | + b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class) | |
22 | + .childHandler(new ChannelInitializer<SocketChannel>() { | |
23 | + @Override | |
24 | + public void initChannel(SocketChannel ch) throws Exception { | |
25 | + // server端发送的是httpResponse,所以要使用HttpResponseEncoder进行编码 | |
26 | + ch.pipeline().addLast(new HttpResponseEncoder()); | |
27 | + // server端接收到的是httpRequest,所以要使用HttpRequestDecoder进行解码 | |
28 | + ch.pipeline().addLast(new HttpRequestDecoder()); | |
29 | + ch.pipeline().addLast(new HttpServerHandler()); | |
30 | + } | |
31 | + }) | |
32 | + .option(ChannelOption.SO_BACKLOG, 128) | |
33 | + .childOption(ChannelOption.SO_KEEPALIVE, true); | |
34 | + | |
35 | + // 绑定端口,开始接收进来的连接 | |
36 | + ChannelFuture f = b.bind(port).sync(); | |
37 | + // 等待服务器 socket 关闭 。 | |
38 | + f.channel().closeFuture().sync(); | |
39 | + } finally { | |
40 | + workerGroup.shutdownGracefully(); | |
41 | + bossGroup.shutdownGracefully(); | |
42 | + } | |
43 | + } | |
44 | +} | |
0 | 45 | \ No newline at end of file | ... | ... |
src/main/java/com/peony/netty/controller/BaseController.java
0 → 100644
src/main/java/com/peony/netty/controller/DocContentController.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/controller/DocContentController.java | |
... | ... | @@ -0,0 +1,32 @@ |
1 | +package com.peony.netty.controller; | |
2 | + | |
3 | +import com.peony.netty.http.Request; | |
4 | +import com.peony.netty.http.Response; | |
5 | +import com.peony.netty.service.DocContentService; | |
6 | +import com.peony.netty.web.Route; | |
7 | +import org.slf4j.Logger; | |
8 | +import org.slf4j.LoggerFactory; | |
9 | + | |
10 | +import static com.peony.netty.web.UriAdapter.get; | |
11 | + | |
12 | +public class DocContentController implements BaseController { | |
13 | + private Logger logger = LoggerFactory.getLogger(DocContentController.class); | |
14 | + | |
15 | + @Override | |
16 | + public void routes() { | |
17 | + final DocContentService docContentService = new DocContentService(); | |
18 | + get("content/:id", new Route() { | |
19 | + @Override | |
20 | + public Object handle(Request req, Response res) { | |
21 | + String id = req.params("id"); | |
22 | + logger.info("获取到的Id: " + id); | |
23 | +// String depress = req.queryParams("depress"); | |
24 | +// if (depress == null) { | |
25 | +// depress = "true"; | |
26 | +// } | |
27 | + String content = docContentService.get(id, true); | |
28 | + return "content " + content; | |
29 | + } | |
30 | + }); | |
31 | + } | |
32 | +} | ... | ... |
src/main/java/com/peony/netty/controller/DocHtmlController.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/controller/DocHtmlController.java | |
... | ... | @@ -0,0 +1,31 @@ |
1 | +package com.peony.netty.controller; | |
2 | + | |
3 | +import com.peony.netty.http.Request; | |
4 | +import com.peony.netty.http.Response; | |
5 | +import com.peony.netty.service.DocHtmlService; | |
6 | +import com.peony.netty.web.Route; | |
7 | +import org.slf4j.Logger; | |
8 | +import org.slf4j.LoggerFactory; | |
9 | + | |
10 | +import static com.peony.netty.web.UriAdapter.get; | |
11 | + | |
12 | +public class DocHtmlController implements BaseController { | |
13 | + private Logger logger = LoggerFactory.getLogger(DocHtmlController.class); | |
14 | + | |
15 | + @Override | |
16 | + public void routes() { | |
17 | + final DocHtmlService docHtmlService = new DocHtmlService(); | |
18 | + get("html/:id", new Route() { | |
19 | + @Override | |
20 | + public Object handle(Request req, Response resp) { | |
21 | + String id = req.params("id"); | |
22 | +// String depress = req.queryParams("depress"); | |
23 | +// if (depress == null) { | |
24 | +// depress = "true"; | |
25 | +// } | |
26 | + String html = docHtmlService.get(id, true); | |
27 | + return "html " + html; | |
28 | + } | |
29 | + }); | |
30 | + } | |
31 | +} | ... | ... |
src/main/java/com/peony/netty/controller/DocUrlController.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/controller/DocUrlController.java | |
... | ... | @@ -0,0 +1,27 @@ |
1 | +package com.peony.netty.controller; | |
2 | + | |
3 | +import com.peony.netty.http.Request; | |
4 | +import com.peony.netty.http.Response; | |
5 | +import com.peony.netty.service.DocUrlService; | |
6 | +import com.peony.netty.web.Route; | |
7 | +import org.slf4j.Logger; | |
8 | +import org.slf4j.LoggerFactory; | |
9 | + | |
10 | +import static com.peony.netty.web.UriAdapter.get; | |
11 | + | |
12 | +public class DocUrlController implements BaseController { | |
13 | + private Logger logger = LoggerFactory.getLogger(DocUrlController.class); | |
14 | + | |
15 | + @Override | |
16 | + public void routes() { | |
17 | + final DocUrlService docUrlService = DocUrlService.getInstance(); | |
18 | + get("url/:id", new Route() { | |
19 | + @Override | |
20 | + public Object handle(Request req, Response resp) { | |
21 | + String id = req.params("id"); | |
22 | + String url = docUrlService.get(id); | |
23 | + return "url " + url; | |
24 | + } | |
25 | + }); | |
26 | + } | |
27 | +} | ... | ... |
src/main/java/com/peony/netty/handler/HttpServerHandler.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/handler/HttpServerHandler.java | |
... | ... | @@ -0,0 +1,49 @@ |
1 | +package com.peony.netty.handler; | |
2 | + | |
3 | +import com.peony.netty.util.ByteBufToBytes; | |
4 | +import com.peony.netty.web.UriAdapter; | |
5 | +import io.netty.channel.ChannelHandlerContext; | |
6 | +import io.netty.channel.ChannelInboundHandlerAdapter; | |
7 | +import io.netty.handler.codec.http.DefaultFullHttpResponse; | |
8 | +import io.netty.handler.codec.http.FullHttpResponse; | |
9 | +import io.netty.handler.codec.http.HttpHeaders; | |
10 | +import io.netty.handler.codec.http.HttpRequest; | |
11 | + | |
12 | +import static io.netty.handler.codec.http.HttpHeaders.Names.*; | |
13 | +import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND; | |
14 | +import static io.netty.handler.codec.http.HttpResponseStatus.OK; | |
15 | +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; | |
16 | + | |
17 | +public class HttpServerHandler extends ChannelInboundHandlerAdapter { | |
18 | + | |
19 | + private ByteBufToBytes reader; | |
20 | + | |
21 | + @Override | |
22 | + public void channelRead(ChannelHandlerContext ctx, Object msg) { | |
23 | + if (msg instanceof HttpRequest) { | |
24 | + HttpRequest request = (HttpRequest) msg; | |
25 | +// System.out.println("User-Agent: " + request.headers().get("User-Agent")); | |
26 | + | |
27 | + FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK); | |
28 | + boolean success = UriAdapter.adapter(request.getUri(), request, response); | |
29 | + if (success) { | |
30 | + response.setStatus(OK); | |
31 | + } else { | |
32 | + response.setStatus(NOT_FOUND); | |
33 | + } | |
34 | + response.headers().set(CONTENT_TYPE, "text/plain; charset=utf-8"); | |
35 | +// response.headers().set(CONTENT_TYPE, "text/html; charset=UTF-8"); | |
36 | + response.headers().set(CONTENT_LENGTH, response.content().readableBytes()); | |
37 | + response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE); | |
38 | + ctx.write(response); | |
39 | + ctx.flush(); | |
40 | + } | |
41 | + } | |
42 | + | |
43 | + @Override | |
44 | + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { | |
45 | + // 当出现异常就关闭连接 | |
46 | + cause.printStackTrace(); | |
47 | + ctx.close(); | |
48 | + } | |
49 | +} | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/http/Request.java | |
... | ... | @@ -0,0 +1,64 @@ |
1 | +package com.peony.netty.http; | |
2 | + | |
3 | +import com.peony.netty.web.PathMatchResult; | |
4 | +import com.peony.util.StringUtils; | |
5 | +import io.netty.handler.codec.http.HttpRequest; | |
6 | + | |
7 | +import java.util.Map; | |
8 | + | |
9 | +public class Request { | |
10 | + | |
11 | + private HttpRequest request; | |
12 | + private Map<String, String> params; | |
13 | + private PathMatchResult pathMatchResult; | |
14 | + private String[] customPaths; | |
15 | + | |
16 | + public Request(HttpRequest request) { | |
17 | + this.request = request; | |
18 | + } | |
19 | + | |
20 | + public HttpRequest getRequest() { | |
21 | + return request; | |
22 | + } | |
23 | + | |
24 | + public void setRequest(HttpRequest request) { | |
25 | + this.request = request; | |
26 | + } | |
27 | + | |
28 | + public Map<String, String> getParams() { | |
29 | + return params; | |
30 | + } | |
31 | + | |
32 | + public void setParams(Map<String, String> params) { | |
33 | + this.params = params; | |
34 | + } | |
35 | + | |
36 | + public PathMatchResult getPathMatchResult() { | |
37 | + return pathMatchResult; | |
38 | + } | |
39 | + | |
40 | + public String[] getCustomPaths() { | |
41 | + return customPaths; | |
42 | + } | |
43 | + | |
44 | + public void setCustomPath(String[] customPaths) { | |
45 | + this.customPaths = customPaths; | |
46 | + } | |
47 | + | |
48 | + public void setPathMatchResult(PathMatchResult pathMatchResult) { | |
49 | + this.pathMatchResult = pathMatchResult; | |
50 | + if (params == null) { | |
51 | + this.params = pathMatchResult.getParams(); | |
52 | + } | |
53 | + if (customPaths == null) { | |
54 | + this.customPaths = pathMatchResult.getCustomPaths(); | |
55 | + } | |
56 | + } | |
57 | + | |
58 | + public String params(String key) { | |
59 | + if (StringUtils.isEmpty(key) || params == null) { | |
60 | + return null; | |
61 | + } | |
62 | + return params.get(key); | |
63 | + } | |
64 | +} | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/http/Response.java | |
... | ... | @@ -0,0 +1,20 @@ |
1 | +package com.peony.netty.http; | |
2 | + | |
3 | +import io.netty.handler.codec.http.HttpResponse; | |
4 | + | |
5 | +public class Response { | |
6 | + | |
7 | + private HttpResponse response; | |
8 | + | |
9 | + public Response(HttpResponse response) { | |
10 | + this.response = response; | |
11 | + } | |
12 | + | |
13 | + public HttpResponse getResponse() { | |
14 | + return response; | |
15 | + } | |
16 | + | |
17 | + public void setResponse(HttpResponse response) { | |
18 | + this.response = response; | |
19 | + } | |
20 | +} | ... | ... |
src/main/java/com/peony/netty/service/DocContentService.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/service/DocContentService.java | |
... | ... | @@ -0,0 +1,32 @@ |
1 | +package com.peony.netty.service; | |
2 | + | |
3 | +import com.peony.netty.util.HbaseFileSystem; | |
4 | +import org.apache.hadoop.hbase.TableName; | |
5 | +import org.slf4j.Logger; | |
6 | +import org.slf4j.LoggerFactory; | |
7 | + | |
8 | +public class DocContentService { | |
9 | + | |
10 | + private static DocContentService instance = new DocContentService(); | |
11 | + private final TableName tableContent = TableName.valueOf("doc"); | |
12 | + private Logger logger = LoggerFactory.getLogger(DocContentService.class); | |
13 | + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance(); | |
14 | + | |
15 | + public static DocContentService getInstance() { | |
16 | + return instance; | |
17 | + } | |
18 | + | |
19 | + public String get(String id, boolean depress) { | |
20 | + String content = null; | |
21 | + try { | |
22 | + content = fileSystem.get(id, tableContent, depress); | |
23 | + if (content == null) { | |
24 | + logger.warn("查询id: " + id + " 正文为空"); | |
25 | + return null; | |
26 | + } | |
27 | + } catch (Exception e) { | |
28 | + logger.error(e.getMessage(), e); | |
29 | + } | |
30 | + return content; | |
31 | + } | |
32 | +} | ... | ... |
src/main/java/com/peony/netty/service/DocHtmlService.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/service/DocHtmlService.java | |
... | ... | @@ -0,0 +1,44 @@ |
1 | +package com.peony.netty.service; | |
2 | + | |
3 | +import com.peony.netty.util.HbaseFileSystem; | |
4 | +import com.peony.util.StringUtils; | |
5 | +import com.qingcloud.qingstor.Qingstor; | |
6 | +import org.apache.hadoop.hbase.TableName; | |
7 | +import org.slf4j.Logger; | |
8 | +import org.slf4j.LoggerFactory; | |
9 | + | |
10 | +public class DocHtmlService { | |
11 | + | |
12 | + private static DocHtmlService instance = new DocHtmlService(); | |
13 | + private final TableName tableContent = TableName.valueOf("doc_html"); | |
14 | + private final String bucketHtml = "doc-html"; | |
15 | + private Logger logger = LoggerFactory.getLogger(DocHtmlService.class); | |
16 | + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance(); | |
17 | + | |
18 | + public static DocHtmlService getInstance() { | |
19 | + return instance; | |
20 | + } | |
21 | + | |
22 | + public String get(String id, boolean depress) { | |
23 | + String html = null; | |
24 | + try { | |
25 | +// html = fileSystem.get(id, tableContent, depress); | |
26 | + } catch (Exception e) { | |
27 | + logger.error(e.getMessage(), e); | |
28 | + } | |
29 | + if (StringUtils.isEmpty(html)) { | |
30 | + try { | |
31 | + String htmlCompress = Qingstor.getObject(bucketHtml, id); | |
32 | + html = depress ? StringUtils.depressString(htmlCompress) : htmlCompress; | |
33 | + } catch (Exception e) { | |
34 | + logger.error(e.getMessage(), e); | |
35 | + } | |
36 | + } | |
37 | + if (html == null) { | |
38 | + logger.warn("查询id: " + id + " HTML为空"); | |
39 | + return null; | |
40 | + } | |
41 | + return html; | |
42 | + } | |
43 | + | |
44 | +} | ... | ... |
src/main/java/com/peony/netty/service/DocUrlService.java
0 → 100644
1 | +++ a/src/main/java/com/peony/netty/service/DocUrlService.java | |
... | ... | @@ -0,0 +1,30 @@ |
1 | +package com.peony.netty.service; | |
2 | + | |
3 | +import com.peony.netty.util.HbaseFileSystem; | |
4 | +import com.peony.util.StringUtils; | |
5 | +import org.apache.hadoop.hbase.TableName; | |
6 | +import org.slf4j.Logger; | |
7 | +import org.slf4j.LoggerFactory; | |
8 | + | |
9 | +public class DocUrlService { | |
10 | + | |
11 | + private static DocUrlService instance = new DocUrlService(); | |
12 | + private final TableName tableUrl = TableName.valueOf("doc_url"); | |
13 | + private Logger logger = LoggerFactory.getLogger(DocUrlService.class); | |
14 | + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance(); | |
15 | + | |
16 | + public static DocUrlService getInstance() { | |
17 | + return instance; | |
18 | + } | |
19 | + | |
20 | + public String get(String id) { | |
21 | + String url = null; | |
22 | + try { | |
23 | + url = new String(fileSystem.get(id, tableUrl), StringUtils.CHARSET); | |
24 | + } catch (Exception e) { | |
25 | + logger.error(e.getMessage(), e); | |
26 | + } | |
27 | + return url; | |
28 | + } | |
29 | + | |
30 | +} | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/util/ByteBufToBytes.java | |
... | ... | @@ -0,0 +1,44 @@ |
1 | +package com.peony.netty.util; | |
2 | + | |
3 | +import io.netty.buffer.ByteBuf; | |
4 | +import io.netty.buffer.Unpooled; | |
5 | + | |
6 | +public class ByteBufToBytes { | |
7 | + private ByteBuf temp; | |
8 | + | |
9 | + private boolean end = true; | |
10 | + | |
11 | + public ByteBufToBytes(int length) { | |
12 | + temp = Unpooled.buffer(length); | |
13 | + } | |
14 | + | |
15 | + public void reading(ByteBuf datas) { | |
16 | + datas.readBytes(temp, datas.readableBytes()); | |
17 | + if (this.temp.writableBytes() != 0) { | |
18 | + end = false; | |
19 | + } else { | |
20 | + end = true; | |
21 | + } | |
22 | + } | |
23 | + | |
24 | + public boolean isEnd() { | |
25 | + return end; | |
26 | + } | |
27 | + | |
28 | + public byte[] readFull() { | |
29 | + if (end) { | |
30 | + byte[] contentByte = new byte[this.temp.readableBytes()]; | |
31 | + this.temp.readBytes(contentByte); | |
32 | + this.temp.release(); | |
33 | + return contentByte; | |
34 | + } else { | |
35 | + return null; | |
36 | + } | |
37 | + } | |
38 | + | |
39 | + public byte[] read(ByteBuf datas) { | |
40 | + byte[] bytes = new byte[datas.readableBytes()]; | |
41 | + datas.readBytes(bytes); | |
42 | + return bytes; | |
43 | + } | |
44 | +} | |
0 | 45 | \ No newline at end of file | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/util/HbaseFileSystem.java | |
... | ... | @@ -0,0 +1,122 @@ |
1 | +package com.peony.netty.util; | |
2 | + | |
3 | +import com.peony.util.StringUtils; | |
4 | +import com.peony.util.SystemProps; | |
5 | +import org.apache.hadoop.conf.Configuration; | |
6 | +import org.apache.hadoop.hbase.HBaseConfiguration; | |
7 | +import org.apache.hadoop.hbase.TableName; | |
8 | +import org.apache.hadoop.hbase.client.*; | |
9 | +import org.apache.hadoop.hbase.util.Bytes; | |
10 | +import org.slf4j.Logger; | |
11 | +import org.slf4j.LoggerFactory; | |
12 | + | |
13 | +import java.io.IOException; | |
14 | +import java.util.ArrayList; | |
15 | +import java.util.List; | |
16 | + | |
17 | +public class HbaseFileSystem { | |
18 | + | |
19 | + private static final Logger logger = LoggerFactory.getLogger(HbaseFileSystem.class); | |
20 | + private static final int BATCH = SystemProps.getIntProperty("batch", 100); | |
21 | + private static final int CACAHING = SystemProps.getIntProperty("caching", 1000); | |
22 | + | |
23 | + private String clientPort = "2181"; | |
24 | + private String quorum = "192.168.3.37,192.168.3.38,192.168.3.39"; | |
25 | + // private String master = "192.168.10.15"; | |
26 | + private String master = "60000"; | |
27 | + // private String parent = "/hbase/hbs-aqtb2h1b"; | |
28 | + private String parent = "/hbase/hbs-gxjpkf8z"; | |
29 | + | |
30 | + private byte[] familyName = Bytes.toBytes("content"); | |
31 | + | |
32 | + private Connection connection; | |
33 | + | |
34 | + private HbaseFileSystem() { | |
35 | + Configuration config = HBaseConfiguration.create(); | |
36 | + config.set("hbase.zookeeper.property.clientPort", SystemProps.getProperty("clientPort", clientPort)); | |
37 | + config.set("hbase.zookeeper.quorum", SystemProps.getProperty("quorum", quorum)); | |
38 | + config.set("hbase.master", SystemProps.getProperty("master", master)); | |
39 | + config.set("zookeeper.znode.parent", SystemProps.getProperty("parent", parent)); | |
40 | + try { | |
41 | + connection = ConnectionFactory.createConnection(config); | |
42 | + } catch (IOException e) { | |
43 | + logger.error("创建到hbase的连接失败", e); | |
44 | + } | |
45 | + } | |
46 | + | |
47 | + public byte[] get(String id, TableName tableName) throws Exception { | |
48 | + Table table = connection.getTable(tableName); | |
49 | + Get get = new Get(Bytes.toBytes(id)); | |
50 | + Result rs = table.get(get); | |
51 | + if (rs.isEmpty()) { | |
52 | + return null; | |
53 | + } | |
54 | + byte[] data = rs.getValue(familyName, null); | |
55 | + if (data == null) { | |
56 | + return null; | |
57 | + } | |
58 | + return data; | |
59 | + } | |
60 | + | |
61 | + public String get(String id, TableName tableName, boolean depress) throws Exception { | |
62 | + byte[] bytes = get(id, tableName); | |
63 | + if (bytes == null) { | |
64 | + return null; | |
65 | + } | |
66 | + if (depress) { | |
67 | + return new String(StringUtils.depress(bytes), StringUtils.CHARSET); | |
68 | + } else { | |
69 | + return StringUtils.encodeBase64String(bytes); | |
70 | + } | |
71 | + } | |
72 | + | |
73 | + public ResultScanner scan(TableName tableName) throws Exception { | |
74 | + Table table = connection.getTable(tableName); | |
75 | + Scan scan = new Scan(); | |
76 | + scan.addFamily(familyName); | |
77 | + scan.setMaxVersions(1); | |
78 | + scan.setCaching(CACAHING); | |
79 | + scan.setBatch(BATCH); | |
80 | + return table.getScanner(scan); | |
81 | + } | |
82 | + | |
83 | + public boolean add(String id, String content, boolean replace, TableName tableName) throws Exception { | |
84 | + Table table = connection.getTable(tableName); | |
85 | + Get get = new Get(Bytes.toBytes(id)); | |
86 | + get.addColumn(familyName, null); | |
87 | + if (!replace && table.exists(get)) { | |
88 | + logger.debug("file exists in hbase, id: {}", id); | |
89 | + return false; | |
90 | + } | |
91 | + Put put = new Put(Bytes.toBytes(id)); | |
92 | + byte[] data = org.apache.commons.codec.binary.StringUtils.getBytesUnchecked(content, StringUtils.CHARSET); | |
93 | + put.addColumn(familyName, null, data); | |
94 | + table.put(put); | |
95 | + return true; | |
96 | + } | |
97 | + | |
98 | + public boolean delete(String id, TableName tableName) throws IOException { | |
99 | + List<String> ids = new ArrayList<>(); | |
100 | + ids.add(id); | |
101 | + return delete(ids, tableName); | |
102 | + } | |
103 | + | |
104 | + public boolean delete(List<String> ids, TableName tableName) throws IOException { | |
105 | + Table table = connection.getTable(tableName); | |
106 | + List<Delete> deletes = new ArrayList<>(); | |
107 | + for (String id : ids) { | |
108 | + Delete delete = new Delete(Bytes.toBytes(id)); | |
109 | + delete.addFamily(familyName); | |
110 | + deletes.add(delete); | |
111 | + } | |
112 | + table.delete(deletes); | |
113 | + return true; | |
114 | + } | |
115 | + | |
116 | + private static final HbaseFileSystem instance = new HbaseFileSystem(); | |
117 | + | |
118 | + public static HbaseFileSystem getInstance() { | |
119 | + return instance; | |
120 | + } | |
121 | + | |
122 | +} | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/web/PathMatchResult.java | |
... | ... | @@ -0,0 +1,33 @@ |
1 | +package com.peony.netty.web; | |
2 | + | |
3 | +import java.util.Map; | |
4 | + | |
5 | +public class PathMatchResult { | |
6 | + private boolean success; | |
7 | + private String[] customPaths; | |
8 | + private Map<String, String> params; | |
9 | + | |
10 | + public boolean success() { | |
11 | + return success; | |
12 | + } | |
13 | + | |
14 | + public void setSuccess(boolean success) { | |
15 | + this.success = success; | |
16 | + } | |
17 | + | |
18 | + public String[] getCustomPaths() { | |
19 | + return customPaths; | |
20 | + } | |
21 | + | |
22 | + public void setCustomPaths(String[] customPaths) { | |
23 | + this.customPaths = customPaths; | |
24 | + } | |
25 | + | |
26 | + public Map<String, String> getParams() { | |
27 | + return params; | |
28 | + } | |
29 | + | |
30 | + public void setParams(Map<String, String> params) { | |
31 | + this.params = params; | |
32 | + } | |
33 | +} | |
0 | 34 | \ No newline at end of file | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/web/Route.java | |
... | ... | @@ -0,0 +1,11 @@ |
1 | +package com.peony.netty.web; | |
2 | + | |
3 | +import com.peony.netty.http.Request; | |
4 | +import com.peony.netty.http.Response; | |
5 | + | |
6 | + | |
7 | +public interface Route { | |
8 | + | |
9 | + Object handle(Request request, Response response); | |
10 | + | |
11 | +} | |
0 | 12 | \ No newline at end of file | ... | ... |
1 | +++ a/src/main/java/com/peony/netty/web/UriAdapter.java | |
... | ... | @@ -0,0 +1,84 @@ |
1 | +package com.peony.netty.web; | |
2 | + | |
3 | +import com.peony.netty.http.Request; | |
4 | +import com.peony.netty.http.Response; | |
5 | +import io.netty.buffer.Unpooled; | |
6 | +import io.netty.handler.codec.http.FullHttpResponse; | |
7 | +import io.netty.handler.codec.http.HttpRequest; | |
8 | +import org.slf4j.Logger; | |
9 | +import org.slf4j.LoggerFactory; | |
10 | + | |
11 | +import java.net.URI; | |
12 | +import java.util.*; | |
13 | + | |
14 | +public class UriAdapter { | |
15 | + | |
16 | + private static Logger logger = LoggerFactory.getLogger(UriAdapter.class); | |
17 | + private static Map<String[], Route> routeMap = new HashMap<>(); | |
18 | + | |
19 | + | |
20 | + public static void get(final String path, final Route route) { | |
21 | + String[] paths = path.split("/"); | |
22 | + routeMap.put(paths, route); | |
23 | + } | |
24 | + | |
25 | + public static boolean adapter(String uri, HttpRequest request, FullHttpResponse response) { | |
26 | + URI u = URI.create(uri); | |
27 | + String path = u.getPath(); | |
28 | + logger.info("path: " + u.getPath() + " query: " + u.getQuery()); | |
29 | + for (Map.Entry<String[], Route> entry : routeMap.entrySet()) { | |
30 | + String[] key = entry.getKey(); | |
31 | + Route route = entry.getValue(); | |
32 | + PathMatchResult pathMatchResult = pathMatch(key, path); | |
33 | + if (pathMatchResult.success()) { | |
34 | + logger.info("匹配成功: " + Arrays.toString(pathMatchResult.getCustomPaths()) + " uri: " + path); | |
35 | + for (Map.Entry<String, String> param : pathMatchResult.getParams().entrySet()) { | |
36 | + logger.info("匹配参数: " + param.getKey() + " : " + param.getValue()); | |
37 | + } | |
38 | + Request req = new Request(request); | |
39 | + Response resp = new Response(response); | |
40 | + req.setPathMatchResult(pathMatchResult); | |
41 | + Object handle = route.handle(req, resp); | |
42 | + if (handle instanceof String) { | |
43 | + String content = (String) handle; | |
44 | + response.content().writeBytes(Unpooled.wrappedBuffer(content.getBytes())); | |
45 | + } | |
46 | + return true; | |
47 | + } | |
48 | + } | |
49 | + logger.warn("匹配失败:" + path); | |
50 | + return false; | |
51 | + } | |
52 | + | |
53 | + | |
54 | + public static PathMatchResult pathMatch(String[] customPaths, String path) { | |
55 | + PathMatchResult pathMatchResult = new PathMatchResult(); | |
56 | + String[] paths = path.substring(1).split("/"); | |
57 | + List<String> pathList = new ArrayList<>(); | |
58 | + for (String subPath: paths) { | |
59 | + pathList.add(subPath); | |
60 | + } | |
61 | + if (pathList.size() != customPaths.length) { | |
62 | + pathMatchResult.setSuccess(false); | |
63 | + return pathMatchResult; | |
64 | + } | |
65 | + Map<String, String> params = new HashMap<>(); | |
66 | + for (int i = 0; i < customPaths.length; i++) { | |
67 | + String s = pathList.get(i); | |
68 | + String customPath = customPaths[i]; | |
69 | + if (customPath.startsWith(":")) { | |
70 | + params.put(customPath.substring(1), s); | |
71 | + } else if (!customPath.equals(s)) { | |
72 | + pathMatchResult.setSuccess(false); | |
73 | + return pathMatchResult; | |
74 | + } | |
75 | + if (i == customPaths.length - 1) { | |
76 | + pathMatchResult.setCustomPaths(customPaths); | |
77 | + pathMatchResult.setParams(params); | |
78 | + } | |
79 | + } | |
80 | + pathMatchResult.setSuccess(true); | |
81 | + return pathMatchResult; | |
82 | + } | |
83 | + | |
84 | +} | ... | ... |
1 | +++ a/src/main/resources/logback.xml | |
... | ... | @@ -0,0 +1,28 @@ |
1 | +<?xml version="1.0" encoding="UTF-8"?> | |
2 | +<configuration> | |
3 | + <property name="CONSOLE_LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss.SSS} %5p --- [%15.15t] %-40.40(%logger{35} %line) : %m%n"/> | |
4 | + | |
5 | + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> | |
6 | + <encoder> | |
7 | + <pattern>${CONSOLE_LOG_PATTERN}</pattern> | |
8 | + <charset>utf8</charset> | |
9 | + </encoder> | |
10 | + </appender> | |
11 | + | |
12 | + <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> | |
13 | + <file>./logs/netty.log</file> | |
14 | + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> | |
15 | + <fileNamePattern>./logs/netty.%d{yyyy-MM-dd}.log</fileNamePattern> | |
16 | + <maxHistory>30</maxHistory> | |
17 | + <totalSizeCap>50GB</totalSizeCap> | |
18 | + </rollingPolicy> | |
19 | + <encoder> | |
20 | + <pattern>${CONSOLE_LOG_PATTERN}</pattern> | |
21 | + </encoder> | |
22 | + </appender> | |
23 | + | |
24 | + <root level="INFO"> | |
25 | + <appender-ref ref="CONSOLE" /> | |
26 | + <appender-ref ref="FILE" /> | |
27 | + </root> | |
28 | +</configuration> | |
0 | 29 | \ No newline at end of file | ... | ... |