crawler-sub / crawler-netty

Download as
Browse Code »

Commit f475e3120003d2449faf3ad1242c20fd9e6d0344

Authored by 辛毅
0 parents
Exists in master

create

Showing 20 changed files with 838 additions and 0 deletions   Show diff stats
.gitignore 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/.gitignore
... ... @@ -0,0 +1,4 @@
  1 +.idea/
  2 +target/
  3 +*.iml
  4 +logs/
... ...
pom.xml 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/pom.xml
... ... @@ -0,0 +1,108 @@
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<project xmlns="http://maven.apache.org/POM/4.0.0"
  3 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 + <modelVersion>4.0.0</modelVersion>
  6 +
  7 + <groupId>com.peony</groupId>
  8 + <artifactId>crawler-netty</artifactId>
  9 + <version>1.0-SNAPSHOT</version>
  10 +
  11 + <dependencies>
  12 + <dependency>
  13 + <groupId>io.netty</groupId>
  14 + <artifactId>netty-all</artifactId>
  15 + <version>4.0.25.Final</version>
  16 + </dependency>
  17 +
  18 + <dependency>
  19 + <groupId>junit</groupId>
  20 + <artifactId>junit</artifactId>
  21 + <version>4.12</version>
  22 + </dependency>
  23 +
  24 + <dependency>
  25 + <groupId>org.slf4j</groupId>
  26 + <artifactId>slf4j-api</artifactId>
  27 + <version>1.7.21</version>
  28 + </dependency>
  29 +
  30 + <dependency>
  31 + <groupId>org.slf4j</groupId>
  32 + <artifactId>log4j-over-slf4j</artifactId>
  33 + <version>1.7.21</version>
  34 + </dependency>
  35 +
  36 + <dependency>
  37 + <groupId>ch.qos.logback</groupId>
  38 + <artifactId>logback-core</artifactId>
  39 + <version>1.1.7</version>
  40 + </dependency>
  41 +
  42 + <dependency>
  43 + <groupId>ch.qos.logback</groupId>
  44 + <artifactId>logback-classic</artifactId>
  45 + <version>1.1.7</version>
  46 + </dependency>
  47 +
  48 + <dependency>
  49 + <groupId>com.peony.crawler</groupId>
  50 + <artifactId>crawler-util</artifactId>
  51 + <version>0.0.1-SNAPSHOT</version>
  52 + </dependency>
  53 +
  54 + <dependency>
  55 + <groupId>org.apache.hbase</groupId>
  56 + <artifactId>hbase-client</artifactId>
  57 + <version>1.2.2</version>
  58 + <exclusions>
  59 + <exclusion>
  60 + <groupId>org.slf4j</groupId>
  61 + <artifactId>slf4j-log4j12</artifactId>
  62 + </exclusion>
  63 + </exclusions>
  64 + </dependency>
  65 +
  66 + <dependency>
  67 + <groupId>com.qingcloud.qingstor</groupId>
  68 + <artifactId>qingstor</artifactId>
  69 + <version>1.0-SNAPSHOT</version>
  70 + </dependency>
  71 + </dependencies>
  72 +
  73 + <build>
  74 + <plugins>
  75 + <plugin>
  76 + <groupId>org.apache.maven.plugins</groupId>
  77 + <artifactId>maven-compiler-plugin</artifactId>
  78 + <configuration>
  79 + <source>1.7</source>
  80 + <target>1.7</target>
  81 + <encoding>UTF-8</encoding>
  82 + </configuration>
  83 + </plugin>
  84 + <plugin>
  85 + <artifactId>maven-assembly-plugin</artifactId>
  86 + <configuration>
  87 + <descriptorRefs>
  88 + <descriptorRef>jar-with-dependencies</descriptorRef>
  89 + </descriptorRefs>
  90 + <archive>
  91 + <manifest>
  92 + <mainClass>com.peony.netty.Application</mainClass>
  93 + </manifest>
  94 + </archive>
  95 + </configuration>
  96 + <executions>
  97 + <execution>
  98 + <id>make-assembly</id>
  99 + <phase>package</phase>
  100 + <goals>
  101 + <goal>single</goal>
  102 + </goals>
  103 + </execution>
  104 + </executions>
  105 + </plugin>
  106 + </plugins>
  107 + </build>
  108 +</project>
0 109 \ No newline at end of file
... ...
src/main/java/com/peony/netty/Application.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/Application.java
... ... @@ -0,0 +1,24 @@
  1 +package com.peony.netty;
  2 +
  3 +import com.peony.netty.controller.DocContentController;
  4 +import com.peony.netty.controller.DocHtmlController;
  5 +import com.peony.netty.http.Request;
  6 +import com.peony.netty.http.Response;
  7 +import com.peony.netty.web.Route;
  8 +
  9 +import static com.peony.netty.web.UriAdapter.get;
  10 +
  11 +public class Application {
  12 + public static void main(String[] args) throws Exception {
  13 + HttpServer server = new HttpServer();
  14 + get("hello", new Route() {
  15 + @Override
  16 + public Object handle(Request request, Response response) {
  17 + return "Hello World 你好";
  18 + }
  19 + });
  20 + new DocContentController().routes();
  21 + new DocHtmlController().routes();
  22 + server.start(8000);
  23 + }
  24 +}
... ...
src/main/java/com/peony/netty/HttpServer.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/HttpServer.java
... ... @@ -0,0 +1,44 @@
  1 +package com.peony.netty;
  2 +
  3 +import com.peony.netty.handler.HttpServerHandler;
  4 +import io.netty.bootstrap.ServerBootstrap;
  5 +import io.netty.channel.ChannelFuture;
  6 +import io.netty.channel.ChannelInitializer;
  7 +import io.netty.channel.ChannelOption;
  8 +import io.netty.channel.EventLoopGroup;
  9 +import io.netty.channel.nio.NioEventLoopGroup;
  10 +import io.netty.channel.socket.SocketChannel;
  11 +import io.netty.channel.socket.nio.NioServerSocketChannel;
  12 +import io.netty.handler.codec.http.HttpRequestDecoder;
  13 +import io.netty.handler.codec.http.HttpResponseEncoder;
  14 +
  15 +public class HttpServer {
  16 + public void start(int port) throws Exception {
  17 + EventLoopGroup bossGroup = new NioEventLoopGroup();
  18 + EventLoopGroup workerGroup = new NioEventLoopGroup();
  19 + try {
  20 + ServerBootstrap b = new ServerBootstrap();
  21 + b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
  22 + .childHandler(new ChannelInitializer<SocketChannel>() {
  23 + @Override
  24 + public void initChannel(SocketChannel ch) throws Exception {
  25 + // server端发送的是httpResponse，所以要使用HttpResponseEncoder进行编码
  26 + ch.pipeline().addLast(new HttpResponseEncoder());
  27 + // server端接收到的是httpRequest，所以要使用HttpRequestDecoder进行解码
  28 + ch.pipeline().addLast(new HttpRequestDecoder());
  29 + ch.pipeline().addLast(new HttpServerHandler());
  30 + }
  31 + })
  32 + .option(ChannelOption.SO_BACKLOG, 128)
  33 + .childOption(ChannelOption.SO_KEEPALIVE, true);
  34 +
  35 + // 绑定端口，开始接收进来的连接
  36 + ChannelFuture f = b.bind(port).sync();
  37 + // 等待服务器 socket 关闭 。
  38 + f.channel().closeFuture().sync();
  39 + } finally {
  40 + workerGroup.shutdownGracefully();
  41 + bossGroup.shutdownGracefully();
  42 + }
  43 + }
  44 +}
0 45 \ No newline at end of file
... ...
src/main/java/com/peony/netty/controller/BaseController.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/controller/BaseController.java
... ... @@ -0,0 +1,7 @@
  1 +package com.peony.netty.controller;
  2 +
  3 +interface BaseController {
  4 +
  5 + void routes();
  6 +
  7 +}
... ...
src/main/java/com/peony/netty/controller/DocContentController.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/controller/DocContentController.java
... ... @@ -0,0 +1,32 @@
  1 +package com.peony.netty.controller;
  2 +
  3 +import com.peony.netty.http.Request;
  4 +import com.peony.netty.http.Response;
  5 +import com.peony.netty.service.DocContentService;
  6 +import com.peony.netty.web.Route;
  7 +import org.slf4j.Logger;
  8 +import org.slf4j.LoggerFactory;
  9 +
  10 +import static com.peony.netty.web.UriAdapter.get;
  11 +
  12 +public class DocContentController implements BaseController {
  13 + private Logger logger = LoggerFactory.getLogger(DocContentController.class);
  14 +
  15 + @Override
  16 + public void routes() {
  17 + final DocContentService docContentService = new DocContentService();
  18 + get("content/:id", new Route() {
  19 + @Override
  20 + public Object handle(Request req, Response res) {
  21 + String id = req.params("id");
  22 + logger.info("获取到的Id： " + id);
  23 +// String depress = req.queryParams("depress");
  24 +// if (depress == null) {
  25 +// depress = "true";
  26 +// }
  27 + String content = docContentService.get(id, true);
  28 + return "content " + content;
  29 + }
  30 + });
  31 + }
  32 +}
... ...
src/main/java/com/peony/netty/controller/DocHtmlController.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/controller/DocHtmlController.java
... ... @@ -0,0 +1,31 @@
  1 +package com.peony.netty.controller;
  2 +
  3 +import com.peony.netty.http.Request;
  4 +import com.peony.netty.http.Response;
  5 +import com.peony.netty.service.DocHtmlService;
  6 +import com.peony.netty.web.Route;
  7 +import org.slf4j.Logger;
  8 +import org.slf4j.LoggerFactory;
  9 +
  10 +import static com.peony.netty.web.UriAdapter.get;
  11 +
  12 +public class DocHtmlController implements BaseController {
  13 + private Logger logger = LoggerFactory.getLogger(DocHtmlController.class);
  14 +
  15 + @Override
  16 + public void routes() {
  17 + final DocHtmlService docHtmlService = new DocHtmlService();
  18 + get("html/:id", new Route() {
  19 + @Override
  20 + public Object handle(Request req, Response resp) {
  21 + String id = req.params("id");
  22 +// String depress = req.queryParams("depress");
  23 +// if (depress == null) {
  24 +// depress = "true";
  25 +// }
  26 + String html = docHtmlService.get(id, true);
  27 + return "html " + html;
  28 + }
  29 + });
  30 + }
  31 +}
... ...
src/main/java/com/peony/netty/controller/DocUrlController.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/controller/DocUrlController.java
... ... @@ -0,0 +1,27 @@
  1 +package com.peony.netty.controller;
  2 +
  3 +import com.peony.netty.http.Request;
  4 +import com.peony.netty.http.Response;
  5 +import com.peony.netty.service.DocUrlService;
  6 +import com.peony.netty.web.Route;
  7 +import org.slf4j.Logger;
  8 +import org.slf4j.LoggerFactory;
  9 +
  10 +import static com.peony.netty.web.UriAdapter.get;
  11 +
  12 +public class DocUrlController implements BaseController {
  13 + private Logger logger = LoggerFactory.getLogger(DocUrlController.class);
  14 +
  15 + @Override
  16 + public void routes() {
  17 + final DocUrlService docUrlService = DocUrlService.getInstance();
  18 + get("url/:id", new Route() {
  19 + @Override
  20 + public Object handle(Request req, Response resp) {
  21 + String id = req.params("id");
  22 + String url = docUrlService.get(id);
  23 + return "url " + url;
  24 + }
  25 + });
  26 + }
  27 +}
... ...
src/main/java/com/peony/netty/handler/HttpServerHandler.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/handler/HttpServerHandler.java
... ... @@ -0,0 +1,49 @@
  1 +package com.peony.netty.handler;
  2 +
  3 +import com.peony.netty.util.ByteBufToBytes;
  4 +import com.peony.netty.web.UriAdapter;
  5 +import io.netty.channel.ChannelHandlerContext;
  6 +import io.netty.channel.ChannelInboundHandlerAdapter;
  7 +import io.netty.handler.codec.http.DefaultFullHttpResponse;
  8 +import io.netty.handler.codec.http.FullHttpResponse;
  9 +import io.netty.handler.codec.http.HttpHeaders;
  10 +import io.netty.handler.codec.http.HttpRequest;
  11 +
  12 +import static io.netty.handler.codec.http.HttpHeaders.Names.*;
  13 +import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
  14 +import static io.netty.handler.codec.http.HttpResponseStatus.OK;
  15 +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
  16 +
  17 +public class HttpServerHandler extends ChannelInboundHandlerAdapter {
  18 +
  19 + private ByteBufToBytes reader;
  20 +
  21 + @Override
  22 + public void channelRead(ChannelHandlerContext ctx, Object msg) {
  23 + if (msg instanceof HttpRequest) {
  24 + HttpRequest request = (HttpRequest) msg;
  25 +// System.out.println("User-Agent: " + request.headers().get("User-Agent"));
  26 +
  27 + FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, OK);
  28 + boolean success = UriAdapter.adapter(request.getUri(), request, response);
  29 + if (success) {
  30 + response.setStatus(OK);
  31 + } else {
  32 + response.setStatus(NOT_FOUND);
  33 + }
  34 + response.headers().set(CONTENT_TYPE, "text/plain; charset=utf-8");
  35 +// response.headers().set(CONTENT_TYPE, "text/html; charset=UTF-8");
  36 + response.headers().set(CONTENT_LENGTH, response.content().readableBytes());
  37 + response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
  38 + ctx.write(response);
  39 + ctx.flush();
  40 + }
  41 + }
  42 +
  43 + @Override
  44 + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
  45 + // 当出现异常就关闭连接
  46 + cause.printStackTrace();
  47 + ctx.close();
  48 + }
  49 +}
... ...
src/main/java/com/peony/netty/http/Request.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/http/Request.java
... ... @@ -0,0 +1,64 @@
  1 +package com.peony.netty.http;
  2 +
  3 +import com.peony.netty.web.PathMatchResult;
  4 +import com.peony.util.StringUtils;
  5 +import io.netty.handler.codec.http.HttpRequest;
  6 +
  7 +import java.util.Map;
  8 +
  9 +public class Request {
  10 +
  11 + private HttpRequest request;
  12 + private Map<String, String> params;
  13 + private PathMatchResult pathMatchResult;
  14 + private String[] customPaths;
  15 +
  16 + public Request(HttpRequest request) {
  17 + this.request = request;
  18 + }
  19 +
  20 + public HttpRequest getRequest() {
  21 + return request;
  22 + }
  23 +
  24 + public void setRequest(HttpRequest request) {
  25 + this.request = request;
  26 + }
  27 +
  28 + public Map<String, String> getParams() {
  29 + return params;
  30 + }
  31 +
  32 + public void setParams(Map<String, String> params) {
  33 + this.params = params;
  34 + }
  35 +
  36 + public PathMatchResult getPathMatchResult() {
  37 + return pathMatchResult;
  38 + }
  39 +
  40 + public String[] getCustomPaths() {
  41 + return customPaths;
  42 + }
  43 +
  44 + public void setCustomPath(String[] customPaths) {
  45 + this.customPaths = customPaths;
  46 + }
  47 +
  48 + public void setPathMatchResult(PathMatchResult pathMatchResult) {
  49 + this.pathMatchResult = pathMatchResult;
  50 + if (params == null) {
  51 + this.params = pathMatchResult.getParams();
  52 + }
  53 + if (customPaths == null) {
  54 + this.customPaths = pathMatchResult.getCustomPaths();
  55 + }
  56 + }
  57 +
  58 + public String params(String key) {
  59 + if (StringUtils.isEmpty(key) || params == null) {
  60 + return null;
  61 + }
  62 + return params.get(key);
  63 + }
  64 +}
... ...
src/main/java/com/peony/netty/http/Response.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/http/Response.java
... ... @@ -0,0 +1,20 @@
  1 +package com.peony.netty.http;
  2 +
  3 +import io.netty.handler.codec.http.HttpResponse;
  4 +
  5 +public class Response {
  6 +
  7 + private HttpResponse response;
  8 +
  9 + public Response(HttpResponse response) {
  10 + this.response = response;
  11 + }
  12 +
  13 + public HttpResponse getResponse() {
  14 + return response;
  15 + }
  16 +
  17 + public void setResponse(HttpResponse response) {
  18 + this.response = response;
  19 + }
  20 +}
... ...
src/main/java/com/peony/netty/service/DocContentService.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/service/DocContentService.java
... ... @@ -0,0 +1,32 @@
  1 +package com.peony.netty.service;
  2 +
  3 +import com.peony.netty.util.HbaseFileSystem;
  4 +import org.apache.hadoop.hbase.TableName;
  5 +import org.slf4j.Logger;
  6 +import org.slf4j.LoggerFactory;
  7 +
  8 +public class DocContentService {
  9 +
  10 + private static DocContentService instance = new DocContentService();
  11 + private final TableName tableContent = TableName.valueOf("doc");
  12 + private Logger logger = LoggerFactory.getLogger(DocContentService.class);
  13 + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance();
  14 +
  15 + public static DocContentService getInstance() {
  16 + return instance;
  17 + }
  18 +
  19 + public String get(String id, boolean depress) {
  20 + String content = null;
  21 + try {
  22 + content = fileSystem.get(id, tableContent, depress);
  23 + if (content == null) {
  24 + logger.warn("查询id: " + id + " 正文为空");
  25 + return null;
  26 + }
  27 + } catch (Exception e) {
  28 + logger.error(e.getMessage(), e);
  29 + }
  30 + return content;
  31 + }
  32 +}
... ...
src/main/java/com/peony/netty/service/DocHtmlService.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/service/DocHtmlService.java
... ... @@ -0,0 +1,44 @@
  1 +package com.peony.netty.service;
  2 +
  3 +import com.peony.netty.util.HbaseFileSystem;
  4 +import com.peony.util.StringUtils;
  5 +import com.qingcloud.qingstor.Qingstor;
  6 +import org.apache.hadoop.hbase.TableName;
  7 +import org.slf4j.Logger;
  8 +import org.slf4j.LoggerFactory;
  9 +
  10 +public class DocHtmlService {
  11 +
  12 + private static DocHtmlService instance = new DocHtmlService();
  13 + private final TableName tableContent = TableName.valueOf("doc_html");
  14 + private final String bucketHtml = "doc-html";
  15 + private Logger logger = LoggerFactory.getLogger(DocHtmlService.class);
  16 + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance();
  17 +
  18 + public static DocHtmlService getInstance() {
  19 + return instance;
  20 + }
  21 +
  22 + public String get(String id, boolean depress) {
  23 + String html = null;
  24 + try {
  25 +// html = fileSystem.get(id, tableContent, depress);
  26 + } catch (Exception e) {
  27 + logger.error(e.getMessage(), e);
  28 + }
  29 + if (StringUtils.isEmpty(html)) {
  30 + try {
  31 + String htmlCompress = Qingstor.getObject(bucketHtml, id);
  32 + html = depress ? StringUtils.depressString(htmlCompress) : htmlCompress;
  33 + } catch (Exception e) {
  34 + logger.error(e.getMessage(), e);
  35 + }
  36 + }
  37 + if (html == null) {
  38 + logger.warn("查询id: " + id + " HTML为空");
  39 + return null;
  40 + }
  41 + return html;
  42 + }
  43 +
  44 +}
... ...
src/main/java/com/peony/netty/service/DocUrlService.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/service/DocUrlService.java
... ... @@ -0,0 +1,30 @@
  1 +package com.peony.netty.service;
  2 +
  3 +import com.peony.netty.util.HbaseFileSystem;
  4 +import com.peony.util.StringUtils;
  5 +import org.apache.hadoop.hbase.TableName;
  6 +import org.slf4j.Logger;
  7 +import org.slf4j.LoggerFactory;
  8 +
  9 +public class DocUrlService {
  10 +
  11 + private static DocUrlService instance = new DocUrlService();
  12 + private final TableName tableUrl = TableName.valueOf("doc_url");
  13 + private Logger logger = LoggerFactory.getLogger(DocUrlService.class);
  14 + private HbaseFileSystem fileSystem = HbaseFileSystem.getInstance();
  15 +
  16 + public static DocUrlService getInstance() {
  17 + return instance;
  18 + }
  19 +
  20 + public String get(String id) {
  21 + String url = null;
  22 + try {
  23 + url = new String(fileSystem.get(id, tableUrl), StringUtils.CHARSET);
  24 + } catch (Exception e) {
  25 + logger.error(e.getMessage(), e);
  26 + }
  27 + return url;
  28 + }
  29 +
  30 +}
... ...
src/main/java/com/peony/netty/util/ByteBufToBytes.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/util/ByteBufToBytes.java
... ... @@ -0,0 +1,44 @@
  1 +package com.peony.netty.util;
  2 +
  3 +import io.netty.buffer.ByteBuf;
  4 +import io.netty.buffer.Unpooled;
  5 +
  6 +public class ByteBufToBytes {
  7 + private ByteBuf temp;
  8 +
  9 + private boolean end = true;
  10 +
  11 + public ByteBufToBytes(int length) {
  12 + temp = Unpooled.buffer(length);
  13 + }
  14 +
  15 + public void reading(ByteBuf datas) {
  16 + datas.readBytes(temp, datas.readableBytes());
  17 + if (this.temp.writableBytes() != 0) {
  18 + end = false;
  19 + } else {
  20 + end = true;
  21 + }
  22 + }
  23 +
  24 + public boolean isEnd() {
  25 + return end;
  26 + }
  27 +
  28 + public byte[] readFull() {
  29 + if (end) {
  30 + byte[] contentByte = new byte[this.temp.readableBytes()];
  31 + this.temp.readBytes(contentByte);
  32 + this.temp.release();
  33 + return contentByte;
  34 + } else {
  35 + return null;
  36 + }
  37 + }
  38 +
  39 + public byte[] read(ByteBuf datas) {
  40 + byte[] bytes = new byte[datas.readableBytes()];
  41 + datas.readBytes(bytes);
  42 + return bytes;
  43 + }
  44 +}
0 45 \ No newline at end of file
... ...
src/main/java/com/peony/netty/util/HbaseFileSystem.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/util/HbaseFileSystem.java
... ... @@ -0,0 +1,122 @@
  1 +package com.peony.netty.util;
  2 +
  3 +import com.peony.util.StringUtils;
  4 +import com.peony.util.SystemProps;
  5 +import org.apache.hadoop.conf.Configuration;
  6 +import org.apache.hadoop.hbase.HBaseConfiguration;
  7 +import org.apache.hadoop.hbase.TableName;
  8 +import org.apache.hadoop.hbase.client.*;
  9 +import org.apache.hadoop.hbase.util.Bytes;
  10 +import org.slf4j.Logger;
  11 +import org.slf4j.LoggerFactory;
  12 +
  13 +import java.io.IOException;
  14 +import java.util.ArrayList;
  15 +import java.util.List;
  16 +
  17 +public class HbaseFileSystem {
  18 +
  19 + private static final Logger logger = LoggerFactory.getLogger(HbaseFileSystem.class);
  20 + private static final int BATCH = SystemProps.getIntProperty("batch", 100);
  21 + private static final int CACAHING = SystemProps.getIntProperty("caching", 1000);
  22 +
  23 + private String clientPort = "2181";
  24 + private String quorum = "192.168.3.37,192.168.3.38,192.168.3.39";
  25 + // private String master = "192.168.10.15";
  26 + private String master = "60000";
  27 + // private String parent = "/hbase/hbs-aqtb2h1b";
  28 + private String parent = "/hbase/hbs-gxjpkf8z";
  29 +
  30 + private byte[] familyName = Bytes.toBytes("content");
  31 +
  32 + private Connection connection;
  33 +
  34 + private HbaseFileSystem() {
  35 + Configuration config = HBaseConfiguration.create();
  36 + config.set("hbase.zookeeper.property.clientPort", SystemProps.getProperty("clientPort", clientPort));
  37 + config.set("hbase.zookeeper.quorum", SystemProps.getProperty("quorum", quorum));
  38 + config.set("hbase.master", SystemProps.getProperty("master", master));
  39 + config.set("zookeeper.znode.parent", SystemProps.getProperty("parent", parent));
  40 + try {
  41 + connection = ConnectionFactory.createConnection(config);
  42 + } catch (IOException e) {
  43 + logger.error("创建到hbase的连接失败", e);
  44 + }
  45 + }
  46 +
  47 + public byte[] get(String id, TableName tableName) throws Exception {
  48 + Table table = connection.getTable(tableName);
  49 + Get get = new Get(Bytes.toBytes(id));
  50 + Result rs = table.get(get);
  51 + if (rs.isEmpty()) {
  52 + return null;
  53 + }
  54 + byte[] data = rs.getValue(familyName, null);
  55 + if (data == null) {
  56 + return null;
  57 + }
  58 + return data;
  59 + }
  60 +
  61 + public String get(String id, TableName tableName, boolean depress) throws Exception {
  62 + byte[] bytes = get(id, tableName);
  63 + if (bytes == null) {
  64 + return null;
  65 + }
  66 + if (depress) {
  67 + return new String(StringUtils.depress(bytes), StringUtils.CHARSET);
  68 + } else {
  69 + return StringUtils.encodeBase64String(bytes);
  70 + }
  71 + }
  72 +
  73 + public ResultScanner scan(TableName tableName) throws Exception {
  74 + Table table = connection.getTable(tableName);
  75 + Scan scan = new Scan();
  76 + scan.addFamily(familyName);
  77 + scan.setMaxVersions(1);
  78 + scan.setCaching(CACAHING);
  79 + scan.setBatch(BATCH);
  80 + return table.getScanner(scan);
  81 + }
  82 +
  83 + public boolean add(String id, String content, boolean replace, TableName tableName) throws Exception {
  84 + Table table = connection.getTable(tableName);
  85 + Get get = new Get(Bytes.toBytes(id));
  86 + get.addColumn(familyName, null);
  87 + if (!replace && table.exists(get)) {
  88 + logger.debug("file exists in hbase, id: {}", id);
  89 + return false;
  90 + }
  91 + Put put = new Put(Bytes.toBytes(id));
  92 + byte[] data = org.apache.commons.codec.binary.StringUtils.getBytesUnchecked(content, StringUtils.CHARSET);
  93 + put.addColumn(familyName, null, data);
  94 + table.put(put);
  95 + return true;
  96 + }
  97 +
  98 + public boolean delete(String id, TableName tableName) throws IOException {
  99 + List<String> ids = new ArrayList<>();
  100 + ids.add(id);
  101 + return delete(ids, tableName);
  102 + }
  103 +
  104 + public boolean delete(List<String> ids, TableName tableName) throws IOException {
  105 + Table table = connection.getTable(tableName);
  106 + List<Delete> deletes = new ArrayList<>();
  107 + for (String id : ids) {
  108 + Delete delete = new Delete(Bytes.toBytes(id));
  109 + delete.addFamily(familyName);
  110 + deletes.add(delete);
  111 + }
  112 + table.delete(deletes);
  113 + return true;
  114 + }
  115 +
  116 + private static final HbaseFileSystem instance = new HbaseFileSystem();
  117 +
  118 + public static HbaseFileSystem getInstance() {
  119 + return instance;
  120 + }
  121 +
  122 +}
... ...
src/main/java/com/peony/netty/web/PathMatchResult.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/web/PathMatchResult.java
... ... @@ -0,0 +1,33 @@
  1 +package com.peony.netty.web;
  2 +
  3 +import java.util.Map;
  4 +
  5 +public class PathMatchResult {
  6 + private boolean success;
  7 + private String[] customPaths;
  8 + private Map<String, String> params;
  9 +
  10 + public boolean success() {
  11 + return success;
  12 + }
  13 +
  14 + public void setSuccess(boolean success) {
  15 + this.success = success;
  16 + }
  17 +
  18 + public String[] getCustomPaths() {
  19 + return customPaths;
  20 + }
  21 +
  22 + public void setCustomPaths(String[] customPaths) {
  23 + this.customPaths = customPaths;
  24 + }
  25 +
  26 + public Map<String, String> getParams() {
  27 + return params;
  28 + }
  29 +
  30 + public void setParams(Map<String, String> params) {
  31 + this.params = params;
  32 + }
  33 +}
0 34 \ No newline at end of file
... ...
src/main/java/com/peony/netty/web/Route.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/web/Route.java
... ... @@ -0,0 +1,11 @@
  1 +package com.peony.netty.web;
  2 +
  3 +import com.peony.netty.http.Request;
  4 +import com.peony.netty.http.Response;
  5 +
  6 +
  7 +public interface Route {
  8 +
  9 + Object handle(Request request, Response response);
  10 +
  11 +}
0 12 \ No newline at end of file
... ...
src/main/java/com/peony/netty/web/UriAdapter.java 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/java/com/peony/netty/web/UriAdapter.java
... ... @@ -0,0 +1,84 @@
  1 +package com.peony.netty.web;
  2 +
  3 +import com.peony.netty.http.Request;
  4 +import com.peony.netty.http.Response;
  5 +import io.netty.buffer.Unpooled;
  6 +import io.netty.handler.codec.http.FullHttpResponse;
  7 +import io.netty.handler.codec.http.HttpRequest;
  8 +import org.slf4j.Logger;
  9 +import org.slf4j.LoggerFactory;
  10 +
  11 +import java.net.URI;
  12 +import java.util.*;
  13 +
  14 +public class UriAdapter {
  15 +
  16 + private static Logger logger = LoggerFactory.getLogger(UriAdapter.class);
  17 + private static Map<String[], Route> routeMap = new HashMap<>();
  18 +
  19 +
  20 + public static void get(final String path, final Route route) {
  21 + String[] paths = path.split("/");
  22 + routeMap.put(paths, route);
  23 + }
  24 +
  25 + public static boolean adapter(String uri, HttpRequest request, FullHttpResponse response) {
  26 + URI u = URI.create(uri);
  27 + String path = u.getPath();
  28 + logger.info("path: " + u.getPath() + " query: " + u.getQuery());
  29 + for (Map.Entry<String[], Route> entry : routeMap.entrySet()) {
  30 + String[] key = entry.getKey();
  31 + Route route = entry.getValue();
  32 + PathMatchResult pathMatchResult = pathMatch(key, path);
  33 + if (pathMatchResult.success()) {
  34 + logger.info("匹配成功: " + Arrays.toString(pathMatchResult.getCustomPaths()) + " uri: " + path);
  35 + for (Map.Entry<String, String> param : pathMatchResult.getParams().entrySet()) {
  36 + logger.info("匹配参数： " + param.getKey() + " : " + param.getValue());
  37 + }
  38 + Request req = new Request(request);
  39 + Response resp = new Response(response);
  40 + req.setPathMatchResult(pathMatchResult);
  41 + Object handle = route.handle(req, resp);
  42 + if (handle instanceof String) {
  43 + String content = (String) handle;
  44 + response.content().writeBytes(Unpooled.wrappedBuffer(content.getBytes()));
  45 + }
  46 + return true;
  47 + }
  48 + }
  49 + logger.warn("匹配失败：" + path);
  50 + return false;
  51 + }
  52 +
  53 +
  54 + public static PathMatchResult pathMatch(String[] customPaths, String path) {
  55 + PathMatchResult pathMatchResult = new PathMatchResult();
  56 + String[] paths = path.substring(1).split("/");
  57 + List<String> pathList = new ArrayList<>();
  58 + for (String subPath: paths) {
  59 + pathList.add(subPath);
  60 + }
  61 + if (pathList.size() != customPaths.length) {
  62 + pathMatchResult.setSuccess(false);
  63 + return pathMatchResult;
  64 + }
  65 + Map<String, String> params = new HashMap<>();
  66 + for (int i = 0; i < customPaths.length; i++) {
  67 + String s = pathList.get(i);
  68 + String customPath = customPaths[i];
  69 + if (customPath.startsWith(":")) {
  70 + params.put(customPath.substring(1), s);
  71 + } else if (!customPath.equals(s)) {
  72 + pathMatchResult.setSuccess(false);
  73 + return pathMatchResult;
  74 + }
  75 + if (i == customPaths.length - 1) {
  76 + pathMatchResult.setCustomPaths(customPaths);
  77 + pathMatchResult.setParams(params);
  78 + }
  79 + }
  80 + pathMatchResult.setSuccess(true);
  81 + return pathMatchResult;
  82 + }
  83 +
  84 +}
... ...
src/main/resources/logback.xml 0 → 100644
  Diff comments   View file @f475e31
  1 +++ a/src/main/resources/logback.xml
... ... @@ -0,0 +1,28 @@
  1 +<?xml version="1.0" encoding="UTF-8"?>
  2 +<configuration>
  3 + <property name="CONSOLE_LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss.SSS} %5p --- [%15.15t] %-40.40(%logger{35} %line) : %m%n"/>
  4 +
  5 + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
  6 + <encoder>
  7 + <pattern>${CONSOLE_LOG_PATTERN}</pattern>
  8 + <charset>utf8</charset>
  9 + </encoder>
  10 + </appender>
  11 +
  12 + <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
  13 + <file>./logs/netty.log</file>
  14 + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
  15 + <fileNamePattern>./logs/netty.%d{yyyy-MM-dd}.log</fileNamePattern>
  16 + <maxHistory>30</maxHistory>
  17 + <totalSizeCap>50GB</totalSizeCap>
  18 + </rollingPolicy>
  19 + <encoder>
  20 + <pattern>${CONSOLE_LOG_PATTERN}</pattern>
  21 + </encoder>
  22 + </appender>
  23 +
  24 + <root level="INFO">
  25 + <appender-ref ref="CONSOLE" />
  26 + <appender-ref ref="FILE" />
  27 + </root>
  28 +</configuration>
0 29 \ No newline at end of file
... ...