clean up
diff --git a/archive/pfs/chunk/chunk.go b/archive/pfs/chunk/chunk.go
new file mode 100644
index 0000000..aaa1bfb
--- /dev/null
+++ b/archive/pfs/chunk/chunk.go
@@ -0,0 +1,23 @@
+package chunk
+
+import (
+ "io"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type ChunkInfo struct {
+ Status api.ChunkStatus
+ Size int
+ Committed int
+}
+
+type Chunk interface {
+ Stats() (ChunkInfo, error)
+ ReaderAt() io.ReaderAt
+ WriterAt() io.WriterAt
+}
+
+type ChunkFactory interface {
+ New(size int) Chunk
+}
diff --git a/archive/pfs/chunk/file.go b/archive/pfs/chunk/file.go
new file mode 100644
index 0000000..3502a50
--- /dev/null
+++ b/archive/pfs/chunk/file.go
@@ -0,0 +1,49 @@
+package chunk
+
+import (
+ "io"
+ "os"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type ReadOnlyFileChunk struct {
+ f *os.File
+ offset int
+ size int
+}
+
+func NewReadOnlyFileChunk(f *os.File, offset, size int) Chunk {
+ return &ReadOnlyFileChunk{f, offset, size}
+}
+
+func (c *ReadOnlyFileChunk) Stats() (ChunkInfo, error) {
+ return ChunkInfo{
+ Status: api.ChunkStatus_READY,
+ Size: c.size,
+ Committed: c.size}, nil
+}
+
+func (c *ReadOnlyFileChunk) ReaderAt() io.ReaderAt {
+ return &fileReader{c.f}
+}
+
+func (c *ReadOnlyFileChunk) WriterAt() io.WriterAt {
+ return &fileWriter{c.f}
+}
+
+type fileReader struct {
+ f *os.File
+}
+
+func (f *fileReader) ReadAt(b []byte, offset int64) (int, error) {
+ return f.f.ReadAt(b, offset)
+}
+
+type fileWriter struct {
+ f *os.File
+}
+
+func (f *fileWriter) WriteAt(b []byte, offset int64) (int, error) {
+ return f.f.WriteAt(b, offset)
+}
diff --git a/archive/pfs/chunk/in_memory.go b/archive/pfs/chunk/in_memory.go
new file mode 100644
index 0000000..b9b55ec
--- /dev/null
+++ b/archive/pfs/chunk/in_memory.go
@@ -0,0 +1,68 @@
+package chunk
+
+import (
+ "bytes"
+ "errors"
+ "io"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type InMemoryChunk struct {
+ status api.ChunkStatus
+ payload []byte
+ committed int
+}
+
+func (c *InMemoryChunk) Stats() (ChunkInfo, error) {
+ return ChunkInfo{c.status, len(c.payload), c.committed}, nil
+}
+
+func (c *InMemoryChunk) ReaderAt() io.ReaderAt {
+ return bytes.NewReader(c.payload[:c.committed])
+}
+
+func (c *InMemoryChunk) WriterAt() io.WriterAt {
+ return &byteWriter{c}
+}
+
+type byteWriter struct {
+ c *InMemoryChunk
+}
+
+func (w *byteWriter) WriteAt(p []byte, offset int64) (n int, err error) {
+ if int(offset) > w.c.committed {
+ panic(1)
+ return 0, errors.New("Gaps are not allowed when writing in chunks")
+ }
+ if int(offset) < w.c.committed {
+ if int(offset)+len(p) <= w.c.committed {
+ if bytes.Compare(w.c.payload[int(offset):int(offset)+len(p)], p) != 0 {
+ panic(2)
+ return 0, errors.New("Can not change contents of allready committed chunk bytes")
+ }
+ panic(3)
+ return len(p), nil
+ }
+ n = w.c.committed - int(offset)
+ p = p[n:]
+ offset = int64(w.c.committed)
+ }
+ if w.c.committed+len(p) > len(w.c.payload) {
+ panic(4)
+ return 0, errors.New("In memory chunk does not have enough space available")
+ }
+ n += copy(w.c.payload[w.c.committed:], p)
+ w.c.committed += n
+ return
+}
+
+type InMemoryChunkFactory struct {
+}
+
+func (f InMemoryChunkFactory) New(size int) Chunk {
+ return &InMemoryChunk{
+ status: api.ChunkStatus_CREATED,
+ payload: make([]byte, size),
+ committed: 0}
+}
diff --git a/archive/pfs/chunk/in_memory_test.go b/archive/pfs/chunk/in_memory_test.go
new file mode 100644
index 0000000..b9711ca
--- /dev/null
+++ b/archive/pfs/chunk/in_memory_test.go
@@ -0,0 +1,27 @@
+package chunk
+
+import (
+ "bytes"
+ "testing"
+)
+
+func TestConcurrentReads(t *testing.T) {
+ c := InMemoryChunkFactory{}.New(4)
+ if _, err := c.WriterAt().WriteAt([]byte("abcd"), 0); err != nil {
+ panic(err)
+ }
+ d1 := make([]byte, 2)
+ d2 := make([]byte, 3)
+ if _, err := c.ReaderAt().ReadAt(d1, 0); err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(d1, []byte("ab")) != 0 {
+ t.Errorf("Expected: %s\nActual: %s", "ab", d1)
+ }
+ if _, err := c.ReaderAt().ReadAt(d2, 0); err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(d2, []byte("abc")) != 0 {
+ t.Errorf("Expected: %s\nActual: %s", "abc", d2)
+ }
+}
diff --git a/archive/pfs/chunk/remote.go b/archive/pfs/chunk/remote.go
new file mode 100644
index 0000000..6d84241
--- /dev/null
+++ b/archive/pfs/chunk/remote.go
@@ -0,0 +1,74 @@
+package chunk
+
+import (
+ "context"
+ "io"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type RemoteChunk struct {
+ chunkId string
+ client api.ChunkStorageClient
+}
+
+func (r *RemoteChunk) Stats() (info ChunkInfo, err error) {
+ resp, err := r.client.GetChunkStatus(
+ context.Background(),
+ &api.GetChunkStatusRequest{ChunkId: r.chunkId})
+ if err != nil {
+ return
+ }
+ info = ChunkInfo{
+ resp.Status,
+ int(resp.TotalBytes),
+ int(resp.CommittedBytes)}
+ return
+}
+
+func (r *RemoteChunk) ReaderAt() io.ReaderAt {
+ return &remoteChunkReaderAt{
+ chunkId: r.chunkId,
+ client: r.client}
+}
+
+func (r *RemoteChunk) WriterAt() io.WriterAt {
+ return &remoteChunkWriterAt{
+ chunkId: r.chunkId,
+ client: r.client}
+}
+
+type remoteChunkReaderAt struct {
+ chunkId string
+ client api.ChunkStorageClient
+}
+
+func (c *remoteChunkReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
+ req := api.ReadChunkRequest{
+ ChunkId: c.chunkId,
+ Offset: int32(offset),
+ NumBytes: int32(len(p))}
+ resp, err := c.client.ReadChunk(context.Background(), &req)
+ if err != nil {
+ return
+ }
+ n = copy(p, resp.Data)
+ return
+}
+
+type remoteChunkWriterAt struct {
+ chunkId string
+ client api.ChunkStorageClient
+}
+
+func (c *remoteChunkWriterAt) WriteAt(p []byte, offset int64) (n int, err error) {
+ req := api.WriteChunkRequest{
+ ChunkId: c.chunkId,
+ Offset: int32(offset),
+ Data: p}
+ resp, err := c.client.WriteChunk(context.Background(), &req)
+ if resp != nil {
+ n = int(resp.BytesWritten)
+ }
+ return
+}
diff --git a/archive/pfs/chunk/replicator.go b/archive/pfs/chunk/replicator.go
new file mode 100644
index 0000000..d8990a7
--- /dev/null
+++ b/archive/pfs/chunk/replicator.go
@@ -0,0 +1,116 @@
+package chunk
+
+import (
+ "context"
+ "io"
+
+ "google.golang.org/grpc"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type ReplicaAssignmentChangeListener interface {
+ Primary(chunkId, currentPrimary string) <-chan string
+}
+
+type PrimaryReplicaChangeListener interface {
+ ChunkId() string
+ Address() <-chan string
+}
+
+type NonChangingReplicaAssignment struct {
+}
+
+func (l *NonChangingReplicaAssignment) Primary(chunkId, address string) <-chan string {
+ ch := make(chan string, 1)
+ ch <- address
+ return ch
+}
+
+func replicate(ctx context.Context, dst, src Chunk, done chan<- int) {
+ dstInfo, err := dst.Stats()
+ if err != nil {
+ panic(err)
+ }
+ inp := src.ReaderAt()
+ replicated := dstInfo.Committed
+ out := dst.WriterAt()
+ for {
+ select {
+ default:
+ p := make([]byte, 100)
+ n, err := inp.ReadAt(p, int64(replicated))
+ if n > 0 {
+ m, _ := out.WriteAt(p[:n], int64(replicated))
+ replicated += m
+ }
+ if err == io.EOF {
+ done <- 1
+ return
+ }
+
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func ReplicateFromPrimary(ctx context.Context, chunkId string, dst Chunk, primaryAddressCh <-chan string) {
+ var done chan int
+ var cancel context.CancelFunc = nil
+ for {
+ select {
+ case <-done:
+ return
+ case <-ctx.Done():
+ return
+ case address := <-primaryAddressCh:
+ if cancel != nil {
+ cancel()
+ }
+ var opts []grpc.DialOption
+ opts = append(opts, grpc.WithInsecure())
+ opts = append(opts, grpc.WithBlock())
+ conn, err := grpc.Dial(address, opts...)
+ if err == nil {
+ continue
+ }
+ client := api.NewChunkStorageClient(conn)
+ src := RemoteChunk{chunkId, client}
+ replicatorCtx, cancelFn := context.WithCancel(context.Background())
+ cancel = cancelFn
+ done = make(chan int, 1)
+ go replicate(replicatorCtx, dst, &src, done)
+ }
+ }
+}
+
+func WriteToPrimary(ctx context.Context, chunkId string, src Chunk, primaryAddressCh <-chan string) {
+ var done chan int
+ var cancel context.CancelFunc = nil
+ for {
+ select {
+ case <-done:
+ return
+ case <-ctx.Done():
+ return
+ case address := <-primaryAddressCh:
+ if cancel != nil {
+ cancel()
+ }
+ var opts []grpc.DialOption
+ opts = append(opts, grpc.WithInsecure())
+ opts = append(opts, grpc.WithBlock())
+ conn, err := grpc.Dial(address, opts...)
+ if err != nil {
+ continue
+ }
+ client := api.NewChunkStorageClient(conn)
+ dst := RemoteChunk{chunkId, client}
+ replicatorCtx, cancelFn := context.WithCancel(context.Background())
+ cancel = cancelFn
+ done = make(chan int, 1)
+ go replicate(replicatorCtx, &dst, src, done)
+ }
+ }
+}
diff --git a/archive/pfs/chunk/server.go b/archive/pfs/chunk/server.go
new file mode 100644
index 0000000..d619f13
--- /dev/null
+++ b/archive/pfs/chunk/server.go
@@ -0,0 +1,123 @@
+package chunk
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "sync"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+type ChunkServer struct {
+ factory ChunkFactory
+ assignmentChangeLis ReplicaAssignmentChangeListener
+ chunks sync.Map
+ replicatorCancel sync.Map
+}
+
+func NewChunkServer(factory ChunkFactory,
+ assignmentChangeLis ReplicaAssignmentChangeListener) *ChunkServer {
+ return &ChunkServer{
+ factory: factory,
+ assignmentChangeLis: assignmentChangeLis}
+}
+
+func (s *ChunkServer) ListChunks(
+ ctx context.Context,
+ req *api.ListChunksRequest) (resp *api.ListChunksResponse, err error) {
+ resp = &api.ListChunksResponse{}
+ s.chunks.Range(func(k, v interface{}) bool {
+ resp.ChunkId = append(resp.ChunkId, k.(string))
+ return true
+ })
+ return
+}
+
+func (s *ChunkServer) CreateChunk(
+ ctx context.Context,
+ req *api.CreateChunkRequest) (resp *api.CreateChunkResponse, err error) {
+ chunk := s.factory.New(int(req.Size))
+ s.chunks.Store(req.ChunkId, chunk)
+ switch req.Role {
+ case api.ReplicaRole_SECONDARY:
+ ctx, cancel := context.WithCancel(context.Background())
+ s.replicatorCancel.Store(req.ChunkId, cancel)
+ primaryAddressCh := s.assignmentChangeLis.Primary(
+ req.ChunkId, req.PrimaryAddress)
+ go ReplicateFromPrimary(ctx, req.ChunkId, chunk, primaryAddressCh)
+ case api.ReplicaRole_PRIMARY:
+ {
+ }
+ }
+ resp = &api.CreateChunkResponse{}
+ log.Printf("Created chunk: %s\n", req.ChunkId)
+ return
+
+}
+
+func (s *ChunkServer) GetChunkStatus(
+ ctx context.Context,
+ req *api.GetChunkStatusRequest) (resp *api.GetChunkStatusResponse, err error) {
+ if chunk, ok := s.chunks.Load(req.ChunkId); ok {
+ c := chunk.(Chunk)
+ var info ChunkInfo
+ info, err = c.Stats()
+ if err != nil {
+ return
+ }
+ resp = &api.GetChunkStatusResponse{
+ Status: info.Status,
+ TotalBytes: int32(info.Size),
+ CommittedBytes: int32(info.Committed)}
+ return
+ }
+ return nil, fmt.Errorf("Could not fund chunk: %s", req.ChunkId)
+}
+
+func (s *ChunkServer) ReadChunk(
+ ctx context.Context,
+ req *api.ReadChunkRequest) (resp *api.ReadChunkResponse, err error) {
+ if value, ok := s.chunks.Load(req.ChunkId); ok {
+ chunk := value.(Chunk)
+ b := make([]byte, req.NumBytes)
+ var n int
+ n, err = chunk.ReaderAt().ReadAt(b, int64(req.Offset))
+ if n == 0 {
+ return
+ }
+ return &api.ReadChunkResponse{Data: b[:n]}, nil
+
+ } else {
+ return nil, fmt.Errorf("Chunk not found: %s", req.ChunkId)
+ }
+}
+
+func (s *ChunkServer) WriteChunk(
+ ctx context.Context,
+ req *api.WriteChunkRequest) (resp *api.WriteChunkResponse, err error) {
+ if value, ok := s.chunks.Load(req.ChunkId); ok {
+ chunk := value.(Chunk)
+ var n int
+ n, err = chunk.WriterAt().WriteAt(req.Data, int64(req.Offset))
+ if n == 0 {
+ return
+ }
+ return &api.WriteChunkResponse{BytesWritten: int32(n)}, nil
+
+ } else {
+ return nil, fmt.Errorf("Chunk not found: %s", req.ChunkId)
+ }
+}
+
+func (s *ChunkServer) RemoveChunk(
+ ctx context.Context,
+ req *api.RemoveChunkRequest) (resp *api.RemoveChunkResponse, err error) {
+ if cancel, ok := s.replicatorCancel.Load(req.ChunkId); ok {
+ cancel.(context.CancelFunc)()
+ s.replicatorCancel.Delete(req.ChunkId)
+ }
+ s.chunks.Delete(req.ChunkId)
+ return &api.RemoveChunkResponse{}, nil
+}
diff --git a/archive/pfs/chunk/server_test.go b/archive/pfs/chunk/server_test.go
new file mode 100644
index 0000000..9549a06
--- /dev/null
+++ b/archive/pfs/chunk/server_test.go
@@ -0,0 +1,103 @@
+package chunk
+
+import (
+ "bytes"
+ "context"
+ "testing"
+
+ "github.com/giolekva/pcloud/pfs/api"
+)
+
+func TestStoreChunk(t *testing.T) {
+ s := ChunkServer{factory: &InMemoryChunkFactory{}}
+ _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
+ ChunkId: "foo",
+ Size: 11,
+ Role: api.ReplicaRole_PRIMARY})
+ if err != nil {
+ t.Error(err)
+ }
+ _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
+ ChunkId: "foo",
+ Offset: 0,
+ Data: []byte("hello world")})
+ if err != nil {
+ t.Error(err)
+ }
+}
+
+func TestStoreAndReadChunk(t *testing.T) {
+ s := ChunkServer{factory: &InMemoryChunkFactory{}}
+ _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
+ ChunkId: "foo",
+ Size: 11,
+ Role: api.ReplicaRole_PRIMARY})
+ if err != nil {
+ t.Error(err)
+ }
+ _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
+ ChunkId: "foo",
+ Offset: 0,
+ Data: []byte("hello world")})
+ if err != nil {
+ t.Error(err)
+ }
+ resp, err := s.ReadChunk(context.Background(), &api.ReadChunkRequest{
+ ChunkId: "foo",
+ NumBytes: 100})
+ if err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(resp.Data, []byte("hello world")) != 0 {
+ t.Errorf("Expected: %s\nGot: %s\n", "hello world", resp.Data)
+ }
+}
+
+func TestReadWithOffsets(t *testing.T) {
+ s := ChunkServer{factory: &InMemoryChunkFactory{}}
+ _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
+ ChunkId: "foo",
+ Size: 11,
+ Role: api.ReplicaRole_PRIMARY})
+ if err != nil {
+ t.Error(err)
+ }
+ _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
+ ChunkId: "foo",
+ Offset: 0,
+ Data: []byte("hello world")})
+ if err != nil {
+ t.Error(err)
+ }
+ resp, err := s.ReadChunk(context.Background(), &api.ReadChunkRequest{
+ ChunkId: "foo",
+ Offset: 0,
+ NumBytes: 2})
+ if err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(resp.Data, []byte("he")) != 0 {
+ t.Errorf("Expected: %s\nGot: %s\n", "he", resp.Data)
+ }
+ resp, err = s.ReadChunk(context.Background(), &api.ReadChunkRequest{
+ ChunkId: "foo",
+ Offset: 2,
+ NumBytes: 2})
+ if err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(resp.Data, []byte("ll")) != 0 {
+ t.Errorf("Expected: %s\nGot: %s\n", "ll", resp.Data)
+ }
+ resp, err = s.ReadChunk(context.Background(), &api.ReadChunkRequest{
+ ChunkId: "foo",
+ Offset: 4,
+ NumBytes: 100})
+ if err != nil {
+ t.Error(err)
+ }
+ if bytes.Compare(resp.Data, []byte("o world")) != 0 {
+ t.Errorf("Expected: %s\nGot: %s\n", "o world", resp.Data)
+ }
+
+}