xd
/
distributed-object-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update pom.xml

This commit is contained in:
Xudong 2022-06-06 16:00:16 +08:00
parent ab19eeebf8
commit b014bcdc85
4 changed files with 56 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api-service/pom.xml
View File

@ -43,6 +43,11 @@
<artifactId>mysql-connector-java</artifactId>
<version>8.0.29</version>
</dependency>
<dependency>
<groupId>com.github.l42111996</groupId>
<artifactId>kcp-base</artifactId>
<version>1.6</version>
</dependency>
</dependencies>
<build>

17
api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/ApiServiceApplication.java
View File

@ -1,6 +1,7 @@
package cn.edu.zjvtit.cloudstorage.apiservice;
import cn.edu.zjvtit.cloudstorage.apiservice.store.DataServerStore;
import com.backblaze.erasure.ReedSolomon;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
@ -23,7 +24,21 @@ public class ApiServiceApplication extends SpringBootServletInitializer {
private static final Logger logger = LoggerFactory.getLogger(ApiServiceApplication.class);
public static void main(String[] args) {
SpringApplication.run(ApiServiceApplication.class, args);
ReedSolomon codec = ReedSolomon.create(4,2);
byte[][] shards = new byte[6][];
shards[0] = new byte[]{0, 1};
shards[1] = new byte[]{4, 5};
shards[2] = new byte[2]; //2,3
shards[3] = new byte[2]; //6,7
shards[4] = new byte[]{44,45};
shards[5] = new byte[]{40,41};
boolean[] shardPresent = new boolean[]{true,true,false,false,true,true};
codec.decodeMissing(shards,shardPresent,0,2);
for(byte[] shard:shards){
System.out.println(shard[0]);
System.out.println(shard[1]);
}
}
@Bean

48
api-service/src/main/java/cn/edu/zjvtit/cloudstorage/apiservice/controller/ObjectsController.java
View File

@ -5,6 +5,9 @@ import cn.edu.zjvtit.cloudstorage.apiservice.entity.ObjectEntity;
import cn.edu.zjvtit.cloudstorage.apiservice.rabbitmq.topic.Sender;
import cn.edu.zjvtit.cloudstorage.apiservice.repository.ObjectRepository;
import cn.edu.zjvtit.cloudstorage.apiservice.store.DataServerStore;
import com.backblaze.erasure.ByteInputOutputExpCodingLoop;
import com.backblaze.erasure.CodingLoop;
import com.backblaze.erasure.ReedSolomon;
import com.rabbitmq.client.Channel;
import org.slf4j.Logger;
import org.springframework.amqp.core.BindingBuilder;
@ -36,6 +39,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.logging.Level;
import static java.util.Arrays.copyOfRange;
@RestController
public class ObjectsController {
@Autowired
@ -63,13 +68,7 @@ public class ObjectsController {
}
String md5 = headers.get("md5");
ObjectEntity object = this.objectRepository.findFirstByNameOrderByVersionDesc(filename);
if (object == null) {
object = new ObjectEntity();
object.setName(filename);
object.setHash(md5);
object.setVersion(1);
object.setSize(requestBody.length);
} else {
if (object != null) {
if (md5.equals(object.getHash())) {
return new Result("object already exists", "10001").toString();
}
@ -79,25 +78,48 @@ public class ObjectsController {
object.setHash(md5);
object.setVersion(version);
object.setSize(requestBody.length);
} else {
object = new ObjectEntity();
object.setName(filename);
object.setHash(md5);
object.setVersion(1);
object.setSize(requestBody.length);
}
int blockSize = (int) Math.ceil(requestBody.length / 4.0);
ReedSolomon codec = ReedSolomon.create(4, 2);
byte[][] shards = new byte[6][];
shards[0] = copyOfRange(requestBody, 0, blockSize - 1);
shards[1] = copyOfRange(requestBody, blockSize, 2 * blockSize - 1);
shards[2] = copyOfRange(requestBody, 2 * blockSize, 3 * blockSize - 1);
shards[3] = new byte[blockSize];
for (int i = 3 * blockSize; i < requestBody.length - 1; i++) {
shards[3][i - 3 * blockSize] = requestBody[i];
}
shards[4] = new byte[blockSize];
shards[5] = new byte[blockSize];
codec.encodeParity(shards, 0,blockSize);
// todo storage shards
this.objectRepository.save(object);
String apiUrl = "http://"+server;
MultiValueMap<String, String> haderWithMd5 = new HttpHeaders();;
String apiUrl = "http://" + server;
MultiValueMap<String, String> haderWithMd5 = new HttpHeaders();
;
haderWithMd5.add("md5", md5);
// 获取uuid
ResponseEntity<Result> ret = this.restTemplate.exchange(apiUrl + "/temp", HttpMethod.POST, new HttpEntity<>(null, haderWithMd5), Result.class);
String uuid = (String) Objects.requireNonNull(ret.getBody()).getData().get("uuid");
// 发送patch请求上传文件
HttpEntity<byte[]> request = new HttpEntity<>(requestBody);
ret = this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.PATCH, request, Result.class);
ret = this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.PATCH, request, Result.class);
String retMd5 = (String) Objects.requireNonNull(ret.getBody()).getData().get("md5");
if (Objects.equals(md5, retMd5)){
if (Objects.equals(md5, retMd5)) {
// 将临时文件转为正式文件
this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.PUT, new HttpEntity<>(null, haderWithMd5), Result.class);
this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.PUT, new HttpEntity<>(null, haderWithMd5), Result.class);
return new Result("success", "10000").toString();
} else {
// 删除文件
this.restTemplate.exchange(apiUrl+"/temp/"+uuid, HttpMethod.DELETE, new HttpEntity<>(null, haderWithMd5), Result.class);
this.restTemplate.exchange(apiUrl + "/temp/" + uuid, HttpMethod.DELETE, new HttpEntity<>(null, haderWithMd5), Result.class);
return new Result("md5 not match", "10002").toString();
}

18
pom.xml
View File

@ -21,27 +21,14 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
@ -63,11 +50,6 @@
<artifactId>spring-rabbit-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>