diff --git a/api-service/pom.xml b/api-service/pom.xml index 67ed083..a6777b8 100644 --- a/api-service/pom.xml +++ b/api-service/pom.xml @@ -43,6 +43,11 @@ mysql-connector-java 8.0.29 + + com.github.l42111996 + kcp-base + 1.6 + diff --git a/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/ApiServiceApplication.java b/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/ApiServiceApplication.java index 3af7c04..790bfa6 100644 --- a/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/ApiServiceApplication.java +++ b/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/ApiServiceApplication.java @@ -1,6 +1,7 @@ package cn.edu.zjvtit.cloudstorage.apiservice; import cn.edu.zjvtit.cloudstorage.apiservice.store.DataServerStore; +import com.backblaze.erasure.ReedSolomon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@ -23,7 +24,21 @@ public class ApiServiceApplication extends SpringBootServletInitializer { private static final Logger logger = LoggerFactory.getLogger(ApiServiceApplication.class); public static void main(String[] args) { - SpringApplication.run(ApiServiceApplication.class, args); + ReedSolomon codec = ReedSolomon.create(4,2); + byte[][] shards = new byte[6][]; + shards[0] = new byte[]{0, 1}; + shards[1] = new byte[]{4, 5}; + shards[2] = new byte[2]; //2,3 + shards[3] = new byte[2]; //6,7 + shards[4] = new byte[]{44,45}; + shards[5] = new byte[]{40,41}; + boolean[] shardPresent = new boolean[]{true,true,false,false,true,true}; + codec.decodeMissing(shards,shardPresent,0,2); + for(byte[] shard:shards){ + System.out.println(shard[0]); + System.out.println(shard[1]); + } + } @Bean diff --git a/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/controller/ObjectsController.java b/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/controller/ObjectsController.java index d4a1faa..e3ef498 100644 --- a/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/controller/ObjectsController.java +++ b/api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/controller/ObjectsController.java @@ -5,6 +5,9 @@ import cn.edu.zjvtit.cloudstorage.apiservice.entity.ObjectEntity; import cn.edu.zjvtit.cloudstorage.apiservice.rabbitmq.topic.Sender; import cn.edu.zjvtit.cloudstorage.apiservice.repository.ObjectRepository; import cn.edu.zjvtit.cloudstorage.apiservice.store.DataServerStore; +import com.backblaze.erasure.ByteInputOutputExpCodingLoop; +import com.backblaze.erasure.CodingLoop; +import com.backblaze.erasure.ReedSolomon; import com.rabbitmq.client.Channel; import org.slf4j.Logger; import org.springframework.amqp.core.BindingBuilder; @@ -36,6 +39,8 @@ import java.util.Map; import java.util.Objects; import java.util.logging.Level; +import static java.util.Arrays.copyOfRange; + @RestController public class ObjectsController { @Autowired @@ -63,13 +68,7 @@ public class ObjectsController { } String md5 = headers.get("md5"); ObjectEntity object = this.objectRepository.findFirstByNameOrderByVersionDesc(filename); - if (object == null) { - object = new ObjectEntity(); - object.setName(filename); - object.setHash(md5); - object.setVersion(1); - object.setSize(requestBody.length); - } else { + if (object != null) { if (md5.equals(object.getHash())) { return new Result("object already exists", "10001").toString(); } @@ -79,25 +78,48 @@ public class ObjectsController { object.setHash(md5); object.setVersion(version); object.setSize(requestBody.length); + } else { + object = new ObjectEntity(); + object.setName(filename); + object.setHash(md5); + object.setVersion(1); + object.setSize(requestBody.length); } + + int blockSize = (int) Math.ceil(requestBody.length / 4.0); + ReedSolomon codec = ReedSolomon.create(4, 2); + byte[][] shards = new byte[6][]; + shards[0] = copyOfRange(requestBody, 0, blockSize - 1); + shards[1] = copyOfRange(requestBody, blockSize, 2 * blockSize - 1); + shards[2] = copyOfRange(requestBody, 2 * blockSize, 3 * blockSize - 1); + shards[3] = new byte[blockSize]; + for (int i = 3 * blockSize; i < requestBody.length - 1; i++) { + shards[3][i - 3 * blockSize] = requestBody[i]; + } + shards[4] = new byte[blockSize]; + shards[5] = new byte[blockSize]; + codec.encodeParity(shards, 0,blockSize); + // todo storage shards + this.objectRepository.save(object); - String apiUrl = "http://"+server; - MultiValueMap haderWithMd5 = new HttpHeaders();; + String apiUrl = "http://" + server; + MultiValueMap haderWithMd5 = new HttpHeaders(); + ; haderWithMd5.add("md5", md5); // 获取uuid ResponseEntity ret = this.restTemplate.exchange(apiUrl + "/temp", HttpMethod.POST, new HttpEntity<>(null, haderWithMd5), Result.class); String uuid = (String) Objects.requireNonNull(ret.getBody()).getData().get("uuid"); // 发送patch请求上传文件 HttpEntity request = new HttpEntity<>(requestBody); - ret = this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.PATCH, request, Result.class); + ret = this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.PATCH, request, Result.class); String retMd5 = (String) Objects.requireNonNull(ret.getBody()).getData().get("md5"); - if (Objects.equals(md5, retMd5)){ + if (Objects.equals(md5, retMd5)) { // 将临时文件转为正式文件 - this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.PUT, new HttpEntity<>(null, haderWithMd5), Result.class); + this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.PUT, new HttpEntity<>(null, haderWithMd5), Result.class); return new Result("success", "10000").toString(); } else { // 删除文件 - this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.DELETE, new HttpEntity<>(null, haderWithMd5), Result.class); + this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.DELETE, new HttpEntity<>(null, haderWithMd5), Result.class); return new Result("md5 not match", "10002").toString(); } diff --git a/pom.xml b/pom.xml index 4db5808..5baacfc 100644 --- a/pom.xml +++ b/pom.xml @@ -21,27 +21,14 @@ org.springframework.boot spring-boot-starter-amqp - - org.springframework.boot - spring-boot-starter-data-elasticsearch - org.springframework.boot spring-boot-starter-data-jpa - - org.springframework.boot - spring-boot-starter-data-mongodb - org.springframework.boot spring-boot-starter-web - - org.springframework.kafka - spring-kafka - - org.springframework.boot spring-boot-devtools @@ -63,11 +50,6 @@ spring-rabbit-test test - - org.springframework.kafka - spring-kafka-test - test -