initial

nerososft · Jul 13, 2018 · f8b3185 · f8b3185
commit f8b3185
Show file tree

Hide file tree

Showing 320 changed files with 8,213 additions and 0 deletions.
diff --git a/bin/config/config.properties b/bin/config/config.properties
@@ -0,0 +1,36 @@
+# 文件块列表文件
+chunk.list-file-path=/Users/neroyang/project/iotcloud/middleware/ndfs/test_data/header.list
+# 文件块存放路径
+chunk.chunk-file-path=/Users/neroyang/project/iotcloud/middleware/ndfs/test_data/
+# 数据节点启动监听端口
+server.host=localhost
+server.port=1080
+# 注册中心地址，若有多个，使用分号分隔
+zookeeper.host=www.cenocloud.com:2181
+# 注册中心超时时间
+zookeeper.timeout=10000
+# 索引节点负载均衡策略 默认轮询
+load-balance.type=polling
+# 节点名称
+node.name=data_01
+# 节点备注
+node.note=data node 01
+# 节点虚拟卷/命名空间
+node.volume=DATA_01
+# 节点类型 Master 主，Slave 备
+node.type=Master
+# 若节点类型为Master，该项配置需要注释，若节点类型为salve，则需要填写需要备份的节点名称
+# node.master = null
+# 数据防篡改， 打开意味着所有对数据的修改和删除操作将失效，并且会产生文件hash链，进行防篡改校验，默认关闭
+data.tamper-proof.enable=false
+# 最大允许上传文件
+data.max-upload-size=5
+# 文件块大小，单位为兆字节，默认64M，参考 64/128/256/... 建议不做修改
+data.chunk-size=64
+# 节点间操作校验秘钥，若配置就意味着与该节点通信的所有节点均需配置，包括namenode，client需带着秘钥访问
+auth.enable=true
+auth.key=2186394109e2427b80710fb561485184
+auth.secret=2186394109e2427b80710fb5614851842186394109e2427b80710fb561485184
+
+
+
diff --git a/bin/ndfsDataServer.sh b/bin/ndfsDataServer.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+$dataNode = "DFSDataServer"
+
+findDataNode(){
+    ${dataNodePid}=`ps -ef | grep "$1" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
+    return ${dataNodePid}
+}
+
+stopDataNode(){
+    kill -9 $1
+}
+
+startDataNode(){
+    nohup sudo java -jar xxx.jar > log/data_node_server.log &
+}
+
diff --git a/bin/ndfsIndexServer.sh b/bin/ndfsIndexServer.sh
@@ -0,0 +1 @@
+#!/usr/bin/env bash
diff --git a/client/config/config.properties b/client/config/config.properties
@@ -0,0 +1,36 @@
+# 文件块列表文件
+chunk.list-file-path=/Users/neroyang/project/iotcloud/middleware/ndfs/test_data/header.list
+# 文件块存放路径
+chunk.chunk-file-path=/Users/neroyang/project/iotcloud/middleware/ndfs/test_data/
+# 数据节点启动监听端口
+server.host=localhost
+server.port=1080
+# 注册中心地址，若有多个，使用分号分隔
+zookeeper.host=www.cenocloud.com:2181
+# 注册中心超时时间
+zookeeper.timeout=10000
+# 索引节点负载均衡策略 默认轮询
+load-balance.type=polling
+# 节点名称
+node.name=data_01
+# 节点备注
+node.note=data node 01
+# 节点虚拟卷/命名空间
+node.volume=DATA_01
+# 节点类型 Master 主，Slave 备
+node.type=Master
+# 若节点类型为Master，该项配置需要注释，若节点类型为salve，则需要填写需要备份的节点名称
+# node.master = null
+# 数据防篡改， 打开意味着所有对数据的修改和删除操作将失效，并且会产生文件hash链，进行防篡改校验，默认关闭
+data.tamper-proof.enable=false
+# 最大允许上传文件
+data.max-upload-size=5
+# 文件块大小，单位为兆字节，默认64M，参考 64/128/256/... 建议不做修改
+data.chunk-size=64
+# 节点间操作校验秘钥，若配置就意味着与该节点通信的所有节点均需配置，包括namenode，client需带着秘钥访问
+auth.enable=true
+auth.key=2186394109e2427b80710fb561485184
+auth.secret=2186394109e2427b80710fb5614851842186394109e2427b80710fb561485184
+
+
+
diff --git a/client/ndfs.client.iml b/client/ndfs.client.iml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7" inherit-compiler-output="false">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="ndfs.common" />
+    <orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.1" level="project" />
+    <orderEntry type="library" name="Maven: org.ow2.util.bundles:zookeeper-3.3.1:1.0.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:zookeeper:3.3.1" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.15" level="project" />
+    <orderEntry type="library" name="Maven: jline:jline:0.9.94" level="project" />
+    <orderEntry type="library" name="Maven: junit:junit:3.8.1" level="project" />
+    <orderEntry type="library" name="Maven: com.github.sgroschupf:zkclient:0.1" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.3.3" level="project" />
+    <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.1.7.2" level="project" />
+    <orderEntry type="library" name="Maven: io.protostuff:protostuff-core:1.4.0" level="project" />
+    <orderEntry type="library" name="Maven: io.protostuff:protostuff-api:1.4.0" level="project" />
+    <orderEntry type="library" name="Maven: io.protostuff:protostuff-runtime:1.4.0" level="project" />
+    <orderEntry type="library" name="Maven: io.protostuff:protostuff-collectionschema:1.4.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
+    <orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.9" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
+    <orderEntry type="library" name="Maven: com.github.oshi:oshi-core:3.4.4" level="project" />
+    <orderEntry type="library" name="Maven: net.java.dev.jna:jna-platform:4.5.0" level="project" />
+    <orderEntry type="library" name="Maven: net.java.dev.jna:jna:4.5.0" level="project" />
+    <orderEntry type="library" name="Maven: org.threeten:threetenbp:1.3.6" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
+    <orderEntry type="library" name="Maven: cglib:cglib:3.2.7" level="project" />
+    <orderEntry type="library" name="Maven: org.ow2.asm:asm:6.2" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.ant:ant:1.10.3" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.10.3" level="project" />
+  </component>
+  <component name="sonarModuleSettings">
+    <option name="alternativeWorkingDirPath" value="" />
+    <option name="localAnalysisScripName" value="&lt;PROJECT&gt;" />
+    <option name="serverName" value="&lt;PROJECT&gt;" />
+    <option name="useAlternativeWorkingDir" value="false" />
+    <option name="workingDirSelection" value="&lt;MODULE&gt;" />
+  </component>
+</module>
diff --git a/client/pom.xml b/client/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <parent>
+        <artifactId>iot_cloud</artifactId>
+        <groupId>com.iot.nero</groupId>
+        <version>1.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>client</artifactId>
+
+    <name>client</name>
+    <!-- FIXME change it to the project's website -->
+    <url>http://www.example.com</url>
+    <dependencies>
+        <dependency>
+            <groupId>com.iot.nero</groupId>
+            <artifactId>middleware.ndfs.common</artifactId>
+            <version>1.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.7</maven.compiler.source>
+        <maven.compiler.target>1.7</maven.compiler.target>
+    </properties>
+
+
+</project>
diff --git a/client/src/main/java/com/iot/nero/middleware/dfs/client/DataDemo.java b/client/src/main/java/com/iot/nero/middleware/dfs/client/DataDemo.java
@@ -0,0 +1,86 @@
+package com.iot.nero.middleware.dfs.client;
+
+import com.iot.nero.middleware.dfs.client.constant.RpcType;
+import com.iot.nero.middleware.dfs.client.core.DFSClient;
+import com.iot.nero.middleware.dfs.client.core.DFSErrorListener;
+import com.iot.nero.middleware.dfs.common.entity.BlockData;
+import com.iot.nero.middleware.dfs.common.entity.FileData;
+import com.iot.nero.middleware.dfs.common.entity.response.Response;
+import com.iot.nero.middleware.dfs.common.service.IFileService;
+import org.xerial.snappy.Snappy;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * Author neroyang
+ * Email  [email protected]
+ * Date   2018/7/4
+ * Time   2:20 PM
+ */
+public class DataDemo {
+    public static void main(String[] args) throws IOException {
+        final DFSClient dfsClient = DFSClient.getInstance();
+        try {
+            dfsClient.init();
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (InstantiationException e) {
+            e.printStackTrace();
+        }
+
+        // 错误事件监听
+        dfsClient.setDfsErrorListener(new DFSErrorListener() {
+
+            @Override
+            public void onError(Response<Object> response) {
+                // 错误处理
+                switch (response.getCode()){
+                    case 1: // 未知的请求类型
+                        System.out.println(response.getMsg());
+                    break;
+                    case 2: // 节点认证失败
+                        System.out.println(response.getMsg());
+                        break;
+                    // todo ....
+                }
+            }
+        });
+
+        final IFileService fileService = dfsClient.getRemoteProxy(RpcType.DATA, IFileService.class);
+
+        BlockData dataPosition = fileService.findFileByHash("hash1");
+        System.out.println(dataPosition);
+
+        try {
+            FileData data1 = fileService.getDataBytes("hash2");
+            byte[] bytes = new byte[data1.getData().size()];
+            for(int i = 0;i<data1.getData().size();i++){
+                bytes[i] = data1.getData().get(i);
+            }
+            try (RandomAccessFile readImg = new RandomAccessFile("/Users/neroyang/project/iotcloud/middleware/ndfs/test_data/steps_received.jpg", "rw")) {
+                if(data1.getCompressMethod().equals("snappy")){
+                    byte[] ok = Snappy.uncompress(bytes);
+                    System.out.println(ok.length);
+                    readImg.write(ok);
+                }
+                readImg.close();
+            } catch (FileNotFoundException e) {
+                e.printStackTrace();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+            System.out.println(data1.getData().size());
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        } catch (InstantiationException e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/client/src/main/java/com/iot/nero/middleware/dfs/client/IndexDemo.java b/client/src/main/java/com/iot/nero/middleware/dfs/client/IndexDemo.java
@@ -0,0 +1,53 @@
+package com.iot.nero.middleware.dfs.client;
+
+import com.iot.nero.middleware.dfs.client.constant.RpcType;
+import com.iot.nero.middleware.dfs.client.core.DFSClient;
+import com.iot.nero.middleware.dfs.client.core.DFSErrorListener;
+import com.iot.nero.middleware.dfs.common.entity.DataPosition;
+import com.iot.nero.middleware.dfs.common.entity.response.Response;
+import com.iot.nero.middleware.dfs.common.service.IIndexService;
+
+import java.io.IOException;
+import java.util.UUID;
+
+/**
+ * Author neroyang
+ * Email  [email protected]
+ * Date   2018/7/4
+ * Time   2:25 PM
+ */
+public class IndexDemo {
+    public static void main(String[] args) throws IOException {
+        final DFSClient dfsClient = DFSClient.getInstance();
+        try {
+            dfsClient.init();
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (InstantiationException e) {
+            e.printStackTrace();
+        }
+
+        // 错误事件监听
+        dfsClient.setDfsErrorListener(new DFSErrorListener() {
+            @Override
+            public void onError(Response<Object> response) {
+                // 错误处理
+                switch (response.getCode()){
+                    case 1: // 未知的请求类型
+                        System.out.println(response.getMsg());
+                    break;
+                    case 2: // 节点认证失败
+                        System.out.println(response.getMsg());
+                        break;
+                    // todo ....
+                }
+            }
+        });
+
+        final IIndexService fileService = dfsClient.getRemoteProxy(RpcType.INDEX, IIndexService.class);
+        DataPosition dataPosition = fileService.getFileByHashCode(UUID.randomUUID().toString());
+        System.out.println(dataPosition);
+    }
+}
diff --git a/client/src/main/java/com/iot/nero/middleware/dfs/client/constant/RpcType.java b/client/src/main/java/com/iot/nero/middleware/dfs/client/constant/RpcType.java
@@ -0,0 +1,11 @@
+package com.iot.nero.middleware.dfs.client.constant;
+
+/**
+ * Author neroyang
+ * Email  [email protected]
+ * Date   2018/6/6
+ * Time   2:25 PM
+ */
+public enum RpcType {
+    DATA,INDEX
+}