Merge pull request #31 from giolekva/docs
Vision doc
#32
diff --git a/.bazelrc b/.bazelrc
index e505d5c..34d0b43 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -1 +1,2 @@
build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64
+# build --sandbox_block_path=
diff --git a/BUILD b/BUILD
index 9027e44..9f3900a 100644
--- a/BUILD
+++ b/BUILD
@@ -1,3 +1,10 @@
+load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
load("@bazel_gazelle//:def.bzl", "gazelle")
-gazelle(name = "gazelle")
+buildifier(
+ name = "buildifier",
+)
+
+gazelle(
+ name = "gazelle",
+)
diff --git a/WORKSPACE b/WORKSPACE
index 43164c1..3bbc45f 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -3,15 +3,18 @@
## Packaging
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
http_archive(
name = "rules_pkg",
+ sha256 = "6b5969a7acd7b60c02f816773b06fcf32fbe8ba0c7919ccdc2df4f8fb923804a",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.3.0/rules_pkg-0.3.0.tar.gz",
"https://github.com/bazelbuild/rules_pkg/releases/download/0.3.0/rules_pkg-0.3.0.tar.gz",
],
- sha256 = "6b5969a7acd7b60c02f816773b06fcf32fbe8ba0c7919ccdc2df4f8fb923804a",
)
+
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
rules_pkg_dependencies()
## Golang
@@ -38,7 +41,9 @@
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
go_rules_dependencies()
+
go_register_toolchains(version = "1.15.5")
+
gazelle_dependencies()
## Docker
@@ -64,23 +69,41 @@
_go_image_repos()
+## Buildifier
+
+http_archive(
+ name = "com_google_protobuf",
+ strip_prefix = "protobuf-master",
+ urls = ["https://github.com/protocolbuffers/protobuf/archive/master.zip"],
+)
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+
+protobuf_deps()
+
+http_archive(
+ name = "com_github_bazelbuild_buildtools",
+ strip_prefix = "buildtools-master",
+ url = "https://github.com/bazelbuild/buildtools/archive/master.zip",
+)
+
## Containier Base Images
load("@io_bazel_rules_docker//container:container.bzl", "container_pull")
container_pull(
name = "alpine_base",
+ digest = "sha256:4e01ddea8def856ba9fee17668fa0b2e45a8bc78127b7ab6cf921f6d6fd86ac9",
registry = "docker.io",
repository = "alpine",
- digest = "sha256:4e01ddea8def856ba9fee17668fa0b2e45a8bc78127b7ab6cf921f6d6fd86ac9",
)
## External Dependencies
go_repository(
name = "com_github_itaysk_regogo",
- importpath = "github.com/itaysk/regogo",
commit = "e9433c1fe5a7c871ce62509fd6376ee114de677a",
+ importpath = "github.com/itaysk/regogo",
)
go_repository(
@@ -91,6 +114,6 @@
go_repository(
name = "com_github_golang_glog",
- importpath = "github.com/golang/glog",
commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998",
+ importpath = "github.com/golang/glog",
)
diff --git a/appmanager/BUILD b/appmanager/BUILD
deleted file mode 100644
index ac84729..0000000
--- a/appmanager/BUILD
+++ /dev/null
@@ -1,32 +0,0 @@
-load("//:bazel_tools/docker.bzl", "docker_image")
-load("//:bazel_tools/helm.bzl", "helm_install")
-load("@rules_pkg//:pkg.bzl", "pkg_tar")
-
-# TODO(lekva): figure out how to build py_binary with pip dependencies and
-# migrate off docker_image rule
-docker_image(
- name = "push_to_dev",
- registry = "localhost:30500",
- image = "giolekva/pcloud-app-manager",
- tag = "latest",
- dockerfile = "Dockerfile",
- srcs = glob(["**"], exclude=["Dockerfile"]),
-)
-
-pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
-)
-
-helm_install(
- name = "install",
- namespace = "pcloud-app-manager",
- release_name = "init",
- chart = ":chart",
- args = {
- "image.name": "localhost:30500/giolekva/pcloud-app-manager",
- "image.pullPolicy": "Always",
- },
-)
diff --git a/apps/face_detection/BUILD b/apps/face_detection/BUILD
index d59b2e6..ebeb5b5 100644
--- a/apps/face_detection/BUILD
+++ b/apps/face_detection/BUILD
@@ -4,19 +4,19 @@
# TODO(lekva): figure out how to build py_binary with pip dependencies and
# migrate off docker_image rule
docker_image(
- name = "push_to_dev",
- registry = "localhost:30500",
- image = "giolekva/face-detector",
- tag = "latest",
- dockerfile = "Dockerfile",
- srcs = glob(["*.py"]) + [
- "haarcascade_frontalface_default.xml",
- ],
+ name = "push_to_dev",
+ srcs = glob(["*.py"]) + [
+ "haarcascade_frontalface_default.xml",
+ ],
+ dockerfile = "Dockerfile",
+ image = "giolekva/face-detector",
+ registry = "localhost:30500",
+ tag = "latest",
)
pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
)
diff --git a/apps/minio/BUILD b/apps/minio/BUILD
index 1fc1992..ad6af19 100644
--- a/apps/minio/BUILD
+++ b/apps/minio/BUILD
@@ -1,8 +1,8 @@
load("@rules_pkg//:pkg.bzl", "pkg_tar")
pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
)
diff --git a/apps/minio/importer/BUILD b/apps/minio/importer/BUILD
index b931f11..09d5da2 100644
--- a/apps/minio/importer/BUILD
+++ b/apps/minio/importer/BUILD
@@ -1,44 +1,44 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-load("@io_bazel_rules_docker//container:container.bzl", "container_push", "container_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
go_library(
- name = "lib",
- srcs = ["importer.go"],
- importpath = "github.com/giolekva/pcloud/apps/minio/importer",
- deps = [
- "@com_github_itaysk_regogo//:go_default_library",
- "@com_github_golang_glog//:go_default_library",
- ],
+ name = "lib",
+ srcs = ["importer.go"],
+ importpath = "github.com/giolekva/pcloud/apps/minio/importer",
+ deps = [
+ "@com_github_golang_glog//:go_default_library",
+ "@com_github_itaysk_regogo//:go_default_library",
+ ],
)
go_binary(
- name = "importer",
- srcs = ["main.go"],
- deps = [
- ":lib",
- ],
+ name = "importer",
+ srcs = ["main.go"],
+ deps = [
+ ":lib",
+ ],
)
container_image(
name = "container",
base = "@alpine_base//image",
- files = [":importer"],
cmd = ["importer"],
+ files = [":importer"],
)
container_push(
- name = "push_to_dev",
- image = ":container",
- format = "Docker",
- registry = "localhost:30500",
- repository = "giolekva/minio-importer",
- tag = "latest",
+ name = "push_to_dev",
+ format = "Docker",
+ image = ":container",
+ registry = "localhost:30500",
+ repository = "giolekva/minio-importer",
+ tag = "latest",
)
pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
)
diff --git a/apps/photos-ui/BUILD b/apps/photos-ui/BUILD
index fa822db..1601870 100644
--- a/apps/photos-ui/BUILD
+++ b/apps/photos-ui/BUILD
@@ -1,55 +1,55 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary")
-load("@io_bazel_rules_docker//container:container.bzl", "container_push", "container_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
filegroup(
name = "data",
srcs = glob([
- "static/**",
- "*.html",
- ]),
+ "static/**",
+ "*.html",
+ ]),
)
go_binary(
- name = "photos_ui",
- srcs = ["main.go"],
- data = [":data"],
- deps = [
- "@io_bazel_rules_go//go/tools/bazel:bazel",
- ],
+ name = "photos_ui",
+ srcs = ["main.go"],
+ data = [":data"],
+ deps = [
+ "@io_bazel_rules_go//go/tools/bazel",
+ ],
)
container_image(
name = "container",
base = "@alpine_base//image",
- files = [
- ":photos_ui",
- ":data",
- "photos_ui.runfiles_manifest",
- ],
- data_path = "/",
cmd = ["/photos_ui"],
+ data_path = "/",
env = {
- "RUNFILES_DIR": "/",
- "RUNFILES_MANIFEST_FILE": "/photos_ui.runfiles_manifest",
+ "RUNFILES_DIR": "/",
+ "RUNFILES_MANIFEST_FILE": "/photos_ui.runfiles_manifest",
},
+ files = [
+ "photos_ui.runfiles_manifest",
+ ":data",
+ ":photos_ui",
+ ],
symlinks = {
- "/photos_ui": "/apps/photos-ui/photos_ui_/photos_ui",
+ "/photos_ui": "/apps/photos-ui/photos_ui_/photos_ui",
},
)
container_push(
- name = "push_to_dev",
- image = ":container",
- format = "Docker",
- registry = "localhost:30500",
- repository = "giolekva/photos-ui",
- tag = "latest",
+ name = "push_to_dev",
+ format = "Docker",
+ image = ":container",
+ registry = "localhost:30500",
+ repository = "giolekva/photos-ui",
+ tag = "latest",
)
pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
)
diff --git a/apps/rpuppy/BUILD b/apps/rpuppy/BUILD
index 4d92193..2e29fc8 100644
--- a/apps/rpuppy/BUILD
+++ b/apps/rpuppy/BUILD
@@ -1,35 +1,35 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary")
-load("@io_bazel_rules_docker//container:container.bzl", "container_push", "container_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
go_binary(
- name = "rpuppy",
- srcs = ["main.go"],
- deps = [
- "@com_github_itaysk_regogo//:go_default_library",
- ],
+ name = "rpuppy",
+ srcs = ["main.go"],
+ deps = [
+ "@com_github_itaysk_regogo//:go_default_library",
+ ],
)
container_image(
name = "container",
# References container_pull from WORKSPACE (above)
base = "@alpine_base//image",
- files = [":rpuppy"],
cmd = ["rpuppy"],
+ files = [":rpuppy"],
)
container_push(
- name = "push_to_dev",
- image = ":container",
- format = "Docker",
- registry = "localhost:30500",
- repository = "giolekva/rpuppy",
- tag = "latest",
+ name = "push_to_dev",
+ format = "Docker",
+ image = ":container",
+ registry = "localhost:30500",
+ repository = "giolekva/rpuppy",
+ tag = "latest",
)
pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
)
diff --git "a/archive/argo/\043argo-events-crds-install.yaml\043" "b/archive/argo/\043argo-events-crds-install.yaml\043"
deleted file mode 100644
index 92b812a..0000000
--- "a/archive/argo/\043argo-events-crds-install.yaml\043"
+++ /dev/null
@@ -1,221 +0,0 @@
-oapiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: gateways.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: Gateway
- listKind: GatewayList
- plural: gateways
- singular: gateway
- shortNames:
- - gw
- scope: Namespaced
- version: "v1alpha1"
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: sensors.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: Sensor
- listKind: SensorList
- plural: sensors
- singular: sensor
- shortNames:
- - sn
- scope: Namespaced
- version: "v1alpha1"
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: eventsources.argoproj.io
-spec:
- group: argoproj.io
- scope: Namespaced
- names:
- kind: EventSource
- plural: eventsources
- singular: eventsource
- listKind: EventSourceList
- shortNames:
- - es
- version: "v1alpha1"
----
-# apiVersion: v1
-# kind: Namespace
-# metadata:
-# name: argo-events
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: argo-events-sa
- namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: argo-events-role-binding
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: argo-events-role
-subjects:
- - kind: ServiceAccount
- name: argo-events-sa
- namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: argo-events-role
-rules:
- - apiGroups:
- - argoproj.io
- verbs:
- - create
- - delete
- - deletecollection
- - get
- - list
- - patch
- - update
- - watch
- resources:
- - workflows
- - workflows/finalizers
- - workflowtemplates
- - workflowtemplates/finalizers
- - gateways
- - gateways/finalizers
- - sensors
- - sensors/finalizers
- - eventsources
- - eventsources/finalizers
- - apiGroups:
- - ""
- resources:
- - pods
- - pods/exec
- - configmaps
- - secrets
- - services
- - events
- - persistentvolumeclaims
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
- - apiGroups:
- - "batch"
- resources:
- - jobs
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
- - apiGroups:
- - "apps"
- resources:
- - deployments
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: gateway-controller-configmap
- namespace: kube-system
-data:
- config: |
- instanceID: argo-events
- namespace: kube-system
----
-# The gateway-controller listens for changes on the gateway CRD and creates gateway
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: gateway-controller
- namespace: kube-system
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: gateway-controller
- template:
- metadata:
- labels:
- app: gateway-controller
- spec:
- serviceAccountName: argo-events-sa
- containers:
- - name: gateway-controller
- image: argoproj/gateway-controller:v0.13.0
- imagePullPolicy: Always
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: CONTROLLER_CONFIG_MAP
- value: gateway-controller-configmap
----
-# The sensor-controller configmap includes configuration information for the sensor-controller
-# To watch sensors created in different namespace than the controller is deployed in, remove the namespace: kube-system.
-# Similarly to watch sensors created in specific namespace, change to namespace: <your_namespace>
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: sensor-controller-configmap
- namespace: kube-system
-data:
- config: |
- instanceID: argo-events
- namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: sensor-controller
- namespace: kube-system
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: sensor-controller
- template:
- metadata:
- labels:
- app: sensor-controller
- spec:
- serviceAccountName: argo-events-sa
- containers:
- - name: sensor-controller
- image: argoproj/sensor-controller:v0.13.0
- imagePullPolicy: Always
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: CONTROLLER_CONFIG_MAP
- value: sensor-controller-configmap
diff --git a/archive/argo/argo-events-crds-install.yaml b/archive/argo/argo-events-crds-install.yaml
deleted file mode 100644
index c746479..0000000
--- a/archive/argo/argo-events-crds-install.yaml
+++ /dev/null
@@ -1,221 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: gateways.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: Gateway
- listKind: GatewayList
- plural: gateways
- singular: gateway
- shortNames:
- - gw
- scope: Namespaced
- version: "v1alpha1"
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: sensors.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: Sensor
- listKind: SensorList
- plural: sensors
- singular: sensor
- shortNames:
- - sn
- scope: Namespaced
- version: "v1alpha1"
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: eventsources.argoproj.io
-spec:
- group: argoproj.io
- scope: Namespaced
- names:
- kind: EventSource
- plural: eventsources
- singular: eventsource
- listKind: EventSourceList
- shortNames:
- - es
- version: "v1alpha1"
----
-# apiVersion: v1
-# kind: Namespace
-# metadata:
-# name: argo-events
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: argo-events-sa
- namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: argo-events-role-binding
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: argo-events-role
-subjects:
- - kind: ServiceAccount
- name: argo-events-sa
- namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: argo-events-role
-rules:
- - apiGroups:
- - argoproj.io
- verbs:
- - create
- - delete
- - deletecollection
- - get
- - list
- - patch
- - update
- - watch
- resources:
- - workflows
- - workflows/finalizers
- - workflowtemplates
- - workflowtemplates/finalizers
- - gateways
- - gateways/finalizers
- - sensors
- - sensors/finalizers
- - eventsources
- - eventsources/finalizers
- - apiGroups:
- - ""
- resources:
- - pods
- - pods/exec
- - configmaps
- - secrets
- - services
- - events
- - persistentvolumeclaims
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
- - apiGroups:
- - "batch"
- resources:
- - jobs
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
- - apiGroups:
- - "apps"
- resources:
- - deployments
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: gateway-controller-configmap
- namespace: kube-system
-data:
- config: |
- instanceID: argo-events
- namespace: kube-system
----
-# The gateway-controller listens for changes on the gateway CRD and creates gateway
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: gateway-controller
- namespace: kube-system
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: gateway-controller
- template:
- metadata:
- labels:
- app: gateway-controller
- spec:
- serviceAccountName: argo-events-sa
- containers:
- - name: gateway-controller
- image: argoproj/gateway-controller:v0.13.0
- imagePullPolicy: Always
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: CONTROLLER_CONFIG_MAP
- value: gateway-controller-configmap
----
-# The sensor-controller configmap includes configuration information for the sensor-controller
-# To watch sensors created in different namespace than the controller is deployed in, remove the namespace: kube-system.
-# Similarly to watch sensors created in specific namespace, change to namespace: <your_namespace>
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: sensor-controller-configmap
- namespace: kube-system
-data:
- config: |
- instanceID: argo-events
- namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: sensor-controller
- namespace: kube-system
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: sensor-controller
- template:
- metadata:
- labels:
- app: sensor-controller
- spec:
- serviceAccountName: argo-events-sa
- containers:
- - name: sensor-controller
- image: argoproj/sensor-controller:v0.13.0
- imagePullPolicy: Always
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: CONTROLLER_CONFIG_MAP
- value: sensor-controller-configmap
diff --git a/archive/argo/event-source.yaml b/archive/argo/event-source.yaml
deleted file mode 100644
index 23ec5ed..0000000
--- a/archive/argo/event-source.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: EventSource
-metadata:
- name: nats-event-source
-spec:
- type: nats
- nats:
- example:
- # url of the nats service
- url: nats://nats.svc:4222
- # jsonBody specifies that all event body payload coming from this
- # source will be JSON
- jsonBody: true
- # subject name
- subject: input-objects
- # optional backoff time for connection retries.
- # if not provided, default connection backoff time will be used.
- connectionBackoff:
- # duration in nanoseconds. following value is 10 seconds
- duration: 10000000000
- # how many backoffs
- steps: 5
- # factor to increase on each step.
- # setting factor > 1 makes backoff exponential.
- factor: 2
- jitter: 0.2
-
-# apiVersion: argoproj.io/v1alpha1
-# kind: EventSource
-# metadata:
-# name: minio-event-source
-# spec:
-# type: "minio"
-# minio:
-# example:
-# bucket:
-# name: input
-# endpoint: mio-minio.default.svc:9000
-# events:
-# - s3:ObjectCreated:Put
-# - s3:ObjectRemoved:Delete
-# insecure: true
-# accessKey:
-# key: accesskey
-# name: artifacts-minio
-# secretKey:
-# key: secretkey
-# name: artifacts-minio
\ No newline at end of file
diff --git a/archive/argo/gateway.yaml b/archive/argo/gateway.yaml
deleted file mode 100644
index b98baf4..0000000
--- a/archive/argo/gateway.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Gateway
-metadata:
- name: nats-gateway
- labels:
- # gateway controller with instanceId "argo-events" will process this gateway
- gateways.argoproj.io/gateway-controller-instanceid: argo-events
-spec:
- type: nats
- eventSourceRef:
- name: nats-event-source
- template:
- metadata:
- name: nats-gateway
- labels:
- gateway-name: nats-gateway
- spec:
- containers:
- - name: gateway-client
- image: argoproj/gateway-client:v0.14.0
- imagePullPolicy: IfNotPresent
- command: ["/bin/gateway-client"]
- - name: nats-events
- image: argoproj/nats-gateway:v0.14.0
- imagePullPolicy: IfNotPresent
- command: ["/bin/nats-gateway"]
- serviceAccountName: argo-events-sa
- subscribers:
- http:
- - "http://nats-sensor.svc:9300/"
-
-
-# apiVersion: argoproj.io/v1alpha1
-# kind: Gateway
-# metadata:
-# name: minio-gateway
-# labels:
-# # gateway controller with instanceId "argo-events" will process this gateway
-# gateways.argoproj.io/gateway-controller-instanceid: argo-events
-# spec:
-# type: minio
-# eventSourceRef:
-# name: minio-event-source
-# template:
-# metadata:
-# name: minio-gateway
-# labels:
-# gateway-name: minio-gateway
-# spec:
-# containers:
-# - name: gateway-client
-# image: argoproj/gateway-client:v0.13.0
-# imagePullPolicy: Always
-# command: ["/bin/gateway-client"]
-# - name: minio-events
-# image: argoproj/artifact-gateway:v0.13.0
-# imagePullPolicy: Always
-# serviceAccountName: argo-events-sa
-# subscribers:
-# http:
-# - "http://minio-sensor.kube-system.svc:9300/"
diff --git a/archive/argo/install.yaml b/archive/argo/install.yaml
deleted file mode 100644
index c800f6a..0000000
--- a/archive/argo/install.yaml
+++ /dev/null
@@ -1,408 +0,0 @@
-# This is an auto-generated file. DO NOT EDIT
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: cronworkflows.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: CronWorkflow
- plural: cronworkflows
- shortNames:
- - cronwf
- - cwf
- scope: Namespaced
- version: v1alpha1
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: workflows.argoproj.io
-spec:
- additionalPrinterColumns:
- - JSONPath: .status.phase
- description: Status of the workflow
- name: Status
- type: string
- - JSONPath: .status.startedAt
- description: When the workflow was started
- format: date-time
- name: Age
- type: date
- group: argoproj.io
- names:
- kind: Workflow
- plural: workflows
- shortNames:
- - wf
- scope: Namespaced
- version: v1alpha1
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
- name: workflowtemplates.argoproj.io
-spec:
- group: argoproj.io
- names:
- kind: WorkflowTemplate
- plural: workflowtemplates
- shortNames:
- - wftmpl
- scope: Namespaced
- version: v1alpha1
-# ---
-# apiVersion: v1
-# kind: ServiceAccount
-# metadata:
-# name: argo
-# ---
-# apiVersion: v1
-# kind: ServiceAccount
-# metadata:
-# name: argo-server
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: argo-role
-rules:
-- apiGroups:
- - ""
- resources:
- - secrets
- verbs:
- - get
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- rbac.authorization.k8s.io/aggregate-to-admin: "true"
- name: argo-aggregate-to-admin
-rules:
-- apiGroups:
- - argoproj.io
- resources:
- - workflows
- - workflows/finalizers
- - workflowtemplates
- - workflowtemplates/finalizers
- - cronworkflows
- - cronworkflows/finalizers
- verbs:
- - create
- - delete
- - deletecollection
- - get
- - list
- - patch
- - update
- - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- rbac.authorization.k8s.io/aggregate-to-edit: "true"
- name: argo-aggregate-to-edit
-rules:
-- apiGroups:
- - argoproj.io
- resources:
- - workflows
- - workflows/finalizers
- - workflowtemplates
- - workflowtemplates/finalizers
- - cronworkflows
- - cronworkflows/finalizers
- verbs:
- - create
- - delete
- - deletecollection
- - get
- - list
- - patch
- - update
- - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- rbac.authorization.k8s.io/aggregate-to-view: "true"
- name: argo-aggregate-to-view
-rules:
-- apiGroups:
- - argoproj.io
- resources:
- - workflows
- - workflows/finalizers
- - workflowtemplates
- - workflowtemplates/finalizers
- - cronworkflows
- - cronworkflows/finalizers
- verbs:
- - get
- - list
- - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: argo-cluster-role
-rules:
-- apiGroups:
- - ""
- resources:
- - pods
- - pods/exec
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
-- apiGroups:
- - ""
- resources:
- - configmaps
- verbs:
- - get
- - watch
- - list
-- apiGroups:
- - ""
- resources:
- - persistentvolumeclaims
- verbs:
- - create
- - delete
-- apiGroups:
- - argoproj.io
- resources:
- - workflows
- - workflows/finalizers
- verbs:
- - get
- - list
- - watch
- - update
- - patch
- - delete
- - create
-- apiGroups:
- - argoproj.io
- resources:
- - workflowtemplates
- - workflowtemplates/finalizers
- verbs:
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - serviceaccounts
- verbs:
- - get
- - list
-- apiGroups:
- - argoproj.io
- resources:
- - cronworkflows
- - cronworkflows/finalizers
- verbs:
- - get
- - list
- - watch
- - update
- - patch
- - delete
-- apiGroups:
- - ""
- resources:
- - events
- verbs:
- - create
-- apiGroups:
- - policy
- resources:
- - poddisruptionbudgets
- verbs:
- - create
- - get
- - delete
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: argo-server-cluster-role
-rules:
-- apiGroups:
- - ""
- resources:
- - configmaps
- verbs:
- - get
- - watch
- - list
-- apiGroups:
- - ""
- resources:
- - secrets
- verbs:
- - get
-- apiGroups:
- - ""
- resources:
- - pods
- - pods/exec
- - pods/log
- verbs:
- - get
- - list
- - watch
- - delete
-- apiGroups:
- - ""
- resources:
- - secrets
- verbs:
- - get
-- apiGroups:
- - argoproj.io
- resources:
- - workflows
- - workflowtemplates
- - cronworkflows
- verbs:
- - create
- - get
- - list
- - watch
- - update
- - patch
- - delete
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: argo-binding
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: argo-role
-subjects:
-- kind: ServiceAccount
- name: default
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: argo-binding
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: argo-cluster-role
-subjects:
-- kind: ServiceAccount
- name: default
- namespace: pcloud
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: argo-server-binding
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: argo-server-cluster-role
-subjects:
-- kind: ServiceAccount
- name: default
- namespace: pcloud
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: workflow-controller-configmap
----
-apiVersion: v1
-kind: Service
-metadata:
- name: argo-server
-spec:
- ports:
- - port: 2746
- targetPort: 2746
- selector:
- app: argo-server
----
-apiVersion: v1
-kind: Service
-metadata:
- name: workflow-controller-metrics
-spec:
- ports:
- - port: 9090
- protocol: TCP
- targetPort: 9090
- selector:
- app: workflow-controller
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: argo-server
-spec:
- selector:
- matchLabels:
- app: argo-server
- template:
- metadata:
- labels:
- app: argo-server
- spec:
- containers:
- - args:
- - server
- image: argoproj/argocli:v2.7.4
- name: argo-server
- ports:
- - containerPort: 2746
- readinessProbe:
- httpGet:
- path: /
- port: 2746
- scheme: HTTP
- initialDelaySeconds: 10
- periodSeconds: 20
- serviceAccountName: default
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: workflow-controller
-spec:
- selector:
- matchLabels:
- app: workflow-controller
- template:
- metadata:
- labels:
- app: workflow-controller
- spec:
- containers:
- - args:
- - --configmap
- - workflow-controller-configmap
- - --executor-image
- - argoproj/argoexec:v2.7.4
- command:
- - workflow-controller
- image: argoproj/workflow-controller:v2.7.4
- name: workflow-controller
- serviceAccountName: default
\ No newline at end of file
diff --git a/archive/argo/mio-minio-secrets.yaml b/archive/argo/mio-minio-secrets.yaml
deleted file mode 100644
index c48d5ef..0000000
--- a/archive/argo/mio-minio-secrets.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
- name: minio-credentials
-data:
- # base64 of minio
- accesskey: bWluaW8K
- # base64 of minio123
- secretkey: bWluaW8xMjMK
diff --git a/archive/argo/sensor.yaml b/archive/argo/sensor.yaml
deleted file mode 100644
index 6918bd5..0000000
--- a/archive/argo/sensor.yaml
+++ /dev/null
@@ -1,138 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Sensor
-metadata:
- name: nats-sensor
- labels:
- sensors.argoproj.io/sensor-controller-instanceid: argo-events
-spec:
- template:
- spec:
- containers:
- - name: sensor
- image: argoproj/sensor:v0.14.0
- imagePullPolicy: IfNotPresent
- serviceAccountName: argo-events-sa
- subscription:
- http:
- port: 9300
- dependencies:
- - name: test-dep
- gatewayName: nats-gateway
- eventName: example
- triggers:
- - template:
- name: nats-workflow-trigger
- k8s:
- group: argoproj.io
- version: v1alpha1
- resource: workflows
- operation: create
- source:
- resource:
- apiVersion: argoproj.io/v1alpha1
- kind: Workflow
- metadata:
- generateName: nats-workflow-
- spec:
- entrypoint: whalesay
- arguments:
- parameters:
- - name: message
- value: WILL_BE_REPLACED
- templates:
- - name: whalesay
- inputs:
- parameters:
- - name: message
- container:
- image: docker/whalesay:latest
- imagePyllPolicy: IfNotPresent
- command: [cowsay]
- args: ["{{inputs.parameters.message}}"]
- parameters:
- - src:
- dependencyName: test-dep
- dest: spec.arguments.parameters.0.value
-
-
-# apiVersion: argoproj.io/v1alpha1
-# kind: Sensor
-# metadata:
-# name: minio-sensor
-# labels:
-# # sensor controller with instanceId "argo-events" will process this sensor
-# sensors.argoproj.io/sensor-controller-instanceid: argo-events
-# spec:
-# template:
-# spec:
-# containers:
-# - name: sensor
-# image: argoproj/sensor:v0.13.0
-# imagePullPolicy: Always
-# serviceAccountName: argo-events-sa
-# subscription:
-# http:
-# port: 9300
-# dependencies:
-# - name: test-dep
-# gatewayName: minio-gateway
-# eventName: example
-# triggers:
-# - template:
-# name: minio-workflow-trigger
-# k8s:
-# group: argoproj.io
-# version: v1alpha1
-# resource: workflows
-# operation: create
-# source:
-# resource:
-# apiVersion: argoproj.io/v1alpha1
-# kind: Workflow
-# metadata:
-# generateName: artifact-workflow-2-
-# spec:
-# entrypoint: detect
-# templates:
-# - name: detect
-# inputs:
-# artifacts:
-# - name: input-image
-# path: /input
-# s3:
-# # endpoint: mio-minio.default.svc:9000
-# # bucket: input # change
-# key: harry.jpg
-# # insecure: true
-# # accessKeySecret:
-# # key: accessKey
-# # name: artifacts-minio
-# # secretKeySecret:
-# # key: secretKey
-# # name: artifacts-minio
-# # useSDKCreds: true
-# # outputs:
-# # artifacts:
-# # - name: output-image
-# # path: /output
-# # s3:
-# # endpoint: mio-minio.default.svc:9000
-# # bucket: output # change
-# # key: PARAMETER
-# # insecure: true
-# # accessKeySecret:
-# # key: accessKey
-# # name: artifacts-minio
-# # secretKeySecret:
-# # key: secretKey
-# # name: artifacts-minio
-# # useSDKCreds: true
-# container:
-# image: face:latest
-# command: [python face.py]
-# args: ["/input", "/output"]
-# # parameters:
-# # - src:
-# # dependencyName: test-dep
-# # dataKey: notification.0.s3.object.key
-# # dest: spec.templates.0.inputs.artifacts.0.s3.key
diff --git a/archive/argo/setup.sh b/archive/argo/setup.sh
deleted file mode 100644
index 7d6cc53..0000000
--- a/archive/argo/setup.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-
-# kubectl create namespace argo
-kubectl apply -n pcloud -f install.yaml
-
-# kubectl apply -n kube-system -f mio-minio-secrets.yaml
-
-
-# helm repo add argo https://argoproj.github.io/argo-helm
-# helm install my-argo --namespace kube-system argo/argo
-# read -s
-# kubectl -n kube-system port-forward deployment/my-argo-server 2746 &
-# read -s
-
-#kubectl apply -n kube-system -f argo-events-crds-install.yaml
-#read -s
-
-
-#kubectl apply -n kube-system -f event-source.yaml
-#kubectl apply -n kube-system -f gateway.yaml
-#kubectl apply -n kube-system -f sensor.yaml
diff --git a/archive/nats/deployment.yaml b/archive/nats/deployment.yaml
deleted file mode 100644
index 66418cb..0000000
--- a/archive/nats/deployment.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-apiVersion: "nats.io/v1alpha2"
-kind: "NatsCluster"
-metadata:
- name: "nats"
-spec:
- size: 1
----
-apiVersion: "streaming.nats.io/v1alpha1"
-kind: "NatsStreamingCluster"
-metadata:
- name: "nats-streaming"
-spec:
- size: 2
- natsSvc: "nats"
\ No newline at end of file
diff --git a/archive/nats/setup.sh b/archive/nats/setup.sh
deleted file mode 100644
index e988ea3..0000000
--- a/archive/nats/setup.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-# kubectl apply -f https://github.com/nats-io/nats-operator/releases/download/v0.5.0/00-prereqs.yaml
-# kubectl apply -f https://github.com/nats-io/nats-operator/releases/download/v0.5.0/10-deployment.yaml
-
-# # Install NATS Streaming Operator on default namespace
-# kubectl apply -f https://raw.githubusercontent.com/nats-io/nats-streaming-operator/master/deploy/default-rbac.yaml
-
-# kubectl apply -f https://raw.githubusercontent.com/nats-io/nats-streaming-operator/master/deploy/deployment.yaml
-
-# sleep 10
-
-kubectl apply -f deployment.yaml
diff --git a/archive/pfs/Dockerfile b/archive/pfs/Dockerfile
deleted file mode 100644
index 5984a5f..0000000
--- a/archive/pfs/Dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-FROM ubuntu:latest
-
-RUN apt-get update --fix-missing
-RUN apt-get -y upgrade
-RUN apt-get -y install wget git bash unzip
-
-WORKDIR /tmp
-RUN wget https://dl.google.com/go/go1.14.linux-amd64.tar.gz
-RUN tar -xvf go1.14.linux-amd64.tar.gz
-RUN mv go /usr/local
-RUN rm go1.14.linux-amd64.tar.gz
-
-ENV GOROOT=/usr/local/go
-ENV GOPATH=/src/go
-ENV GOBIN=$GOPATH/bin
-ENV PATH=$GOBIN:$GOROOT/bin:$PATH
-
-RUN go get -u google.golang.org/grpc
-
-WORKDIR /src/protoc
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
-RUN unzip protoc-3.11.4-linux-x86_64.zip
-RUN rm protoc-3.11.4-linux-x86_64.zip
-ENV PATH=/src/protoc/bin:$PATH
-
-RUN go get -u github.com/golang/protobuf/protoc-gen-go
-RUN go get -u google.golang.org/protobuf/encoding/prototext
-RUN go get -u github.com/google/uuid
-
-WORKDIR /src/go/src/github.com/giolekva/pcloud/pfs
diff --git a/archive/pfs/api/api.proto b/archive/pfs/api/api.proto
deleted file mode 100644
index 56a8e9e..0000000
--- a/archive/pfs/api/api.proto
+++ /dev/null
@@ -1,132 +0,0 @@
-syntax = "proto3";
-
-package pcloud.api;
-
-option go_package = "api";
-
-enum ChunkStatus {
- NEW = 0;
- CREATED = 1;
- WRITING = 2;
- REPLICATING = 3;
- READY = 4;
-}
-
-enum ReplicaRole {
- SECONDARY = 0;
- PRIMARY = 1;
-}
-
-// ChunkStorage
-
-service ChunkStorage {
- rpc ListChunks(ListChunksRequest) returns (ListChunksResponse) {}
-
- rpc CreateChunk(CreateChunkRequest) returns (CreateChunkResponse) {}
-
- rpc GetChunkStatus(GetChunkStatusRequest) returns (GetChunkStatusResponse) {}
-
- rpc ReadChunk(ReadChunkRequest) returns (ReadChunkResponse) {}
-
- rpc WriteChunk(WriteChunkRequest) returns (WriteChunkResponse) {}
-
- rpc RemoveChunk(RemoveChunkRequest) returns (RemoveChunkResponse) {}
-}
-
-message ListChunksRequest {
-}
-
-message ListChunksResponse {
- repeated string chunk_id = 1;
-}
-
-message CreateChunkRequest {
- string chunk_id = 1;
- int32 size = 2;
- ReplicaRole role = 3;
- string primary_address = 4;
-}
-
-message CreateChunkResponse {
-}
-
-message GetChunkStatusRequest {
- string chunk_id = 1;
-}
-
-message GetChunkStatusResponse {
- ChunkStatus status = 1;
- int32 total_bytes = 2;
- int32 committed_bytes = 3;
-}
-
-message ReadChunkRequest {
- string chunk_id = 1;
- int32 offset = 2;
- int32 num_bytes = 3;
-}
-
-message ReadChunkResponse {
- bytes data = 1;
-}
-
-message WriteChunkRequest {
- string chunk_id = 1;
- int32 offset = 2;
- bytes data = 3;
-}
-
-message WriteChunkResponse {
- int32 bytes_written = 1;
-}
-
-message RemoveChunkRequest {
- string chunk_id = 1;
-}
-
-message RemoveChunkResponse {
-}
-
-// MetadataStorage
-
-message ChunkStorageMetadata {
- string chunk_id = 1;
- int32 size_bytes = 2;
- repeated string server = 3;
-}
-
-service MetadataStorage {
- rpc AddChunkServer(AddChunkServerRequest) returns (AddChunkServerResponse) {}
-
- rpc CreateBlob(CreateBlobRequest) returns (CreateBlobResponse) {}
-
- rpc GetBlobMetadata(GetBlobMetadataRequest) returns (GetBlobMetadataResponse) {}
-}
-
-message AddChunkServerRequest {
- string address = 1;
-}
-
-message AddChunkServerResponse {
-}
-
-message CreateBlobRequest {
- int32 size_bytes = 1;
- int32 chunk_size_bytes = 2;
- int32 num_replicas = 3;
-}
-
-message CreateBlobResponse {
- string blob_id = 1;
- repeated ChunkStorageMetadata chunk = 2;
-}
-
-message GetBlobMetadataRequest {
- string blob_id = 1;
-}
-
-message GetBlobMetadataResponse {
- string blob_id = 1;
- int32 size_bytes = 2;
- repeated ChunkStorageMetadata chunk = 3;
-}
\ No newline at end of file
diff --git a/archive/pfs/api/client.go b/archive/pfs/api/client.go
deleted file mode 100644
index 3e05de6..0000000
--- a/archive/pfs/api/client.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package api
-
-import (
- "google.golang.org/grpc"
-)
-
-func DialConn(address string) (*grpc.ClientConn, error) {
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- return grpc.Dial(address, opts...)
-}
diff --git a/archive/pfs/chunk/chunk.go b/archive/pfs/chunk/chunk.go
deleted file mode 100644
index aaa1bfb..0000000
--- a/archive/pfs/chunk/chunk.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package chunk
-
-import (
- "io"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type ChunkInfo struct {
- Status api.ChunkStatus
- Size int
- Committed int
-}
-
-type Chunk interface {
- Stats() (ChunkInfo, error)
- ReaderAt() io.ReaderAt
- WriterAt() io.WriterAt
-}
-
-type ChunkFactory interface {
- New(size int) Chunk
-}
diff --git a/archive/pfs/chunk/file.go b/archive/pfs/chunk/file.go
deleted file mode 100644
index 3502a50..0000000
--- a/archive/pfs/chunk/file.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package chunk
-
-import (
- "io"
- "os"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type ReadOnlyFileChunk struct {
- f *os.File
- offset int
- size int
-}
-
-func NewReadOnlyFileChunk(f *os.File, offset, size int) Chunk {
- return &ReadOnlyFileChunk{f, offset, size}
-}
-
-func (c *ReadOnlyFileChunk) Stats() (ChunkInfo, error) {
- return ChunkInfo{
- Status: api.ChunkStatus_READY,
- Size: c.size,
- Committed: c.size}, nil
-}
-
-func (c *ReadOnlyFileChunk) ReaderAt() io.ReaderAt {
- return &fileReader{c.f}
-}
-
-func (c *ReadOnlyFileChunk) WriterAt() io.WriterAt {
- return &fileWriter{c.f}
-}
-
-type fileReader struct {
- f *os.File
-}
-
-func (f *fileReader) ReadAt(b []byte, offset int64) (int, error) {
- return f.f.ReadAt(b, offset)
-}
-
-type fileWriter struct {
- f *os.File
-}
-
-func (f *fileWriter) WriteAt(b []byte, offset int64) (int, error) {
- return f.f.WriteAt(b, offset)
-}
diff --git a/archive/pfs/chunk/in_memory.go b/archive/pfs/chunk/in_memory.go
deleted file mode 100644
index b9b55ec..0000000
--- a/archive/pfs/chunk/in_memory.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package chunk
-
-import (
- "bytes"
- "errors"
- "io"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type InMemoryChunk struct {
- status api.ChunkStatus
- payload []byte
- committed int
-}
-
-func (c *InMemoryChunk) Stats() (ChunkInfo, error) {
- return ChunkInfo{c.status, len(c.payload), c.committed}, nil
-}
-
-func (c *InMemoryChunk) ReaderAt() io.ReaderAt {
- return bytes.NewReader(c.payload[:c.committed])
-}
-
-func (c *InMemoryChunk) WriterAt() io.WriterAt {
- return &byteWriter{c}
-}
-
-type byteWriter struct {
- c *InMemoryChunk
-}
-
-func (w *byteWriter) WriteAt(p []byte, offset int64) (n int, err error) {
- if int(offset) > w.c.committed {
- panic(1)
- return 0, errors.New("Gaps are not allowed when writing in chunks")
- }
- if int(offset) < w.c.committed {
- if int(offset)+len(p) <= w.c.committed {
- if bytes.Compare(w.c.payload[int(offset):int(offset)+len(p)], p) != 0 {
- panic(2)
- return 0, errors.New("Can not change contents of allready committed chunk bytes")
- }
- panic(3)
- return len(p), nil
- }
- n = w.c.committed - int(offset)
- p = p[n:]
- offset = int64(w.c.committed)
- }
- if w.c.committed+len(p) > len(w.c.payload) {
- panic(4)
- return 0, errors.New("In memory chunk does not have enough space available")
- }
- n += copy(w.c.payload[w.c.committed:], p)
- w.c.committed += n
- return
-}
-
-type InMemoryChunkFactory struct {
-}
-
-func (f InMemoryChunkFactory) New(size int) Chunk {
- return &InMemoryChunk{
- status: api.ChunkStatus_CREATED,
- payload: make([]byte, size),
- committed: 0}
-}
diff --git a/archive/pfs/chunk/in_memory_test.go b/archive/pfs/chunk/in_memory_test.go
deleted file mode 100644
index b9711ca..0000000
--- a/archive/pfs/chunk/in_memory_test.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package chunk
-
-import (
- "bytes"
- "testing"
-)
-
-func TestConcurrentReads(t *testing.T) {
- c := InMemoryChunkFactory{}.New(4)
- if _, err := c.WriterAt().WriteAt([]byte("abcd"), 0); err != nil {
- panic(err)
- }
- d1 := make([]byte, 2)
- d2 := make([]byte, 3)
- if _, err := c.ReaderAt().ReadAt(d1, 0); err != nil {
- t.Error(err)
- }
- if bytes.Compare(d1, []byte("ab")) != 0 {
- t.Errorf("Expected: %s\nActual: %s", "ab", d1)
- }
- if _, err := c.ReaderAt().ReadAt(d2, 0); err != nil {
- t.Error(err)
- }
- if bytes.Compare(d2, []byte("abc")) != 0 {
- t.Errorf("Expected: %s\nActual: %s", "abc", d2)
- }
-}
diff --git a/archive/pfs/chunk/remote.go b/archive/pfs/chunk/remote.go
deleted file mode 100644
index 6d84241..0000000
--- a/archive/pfs/chunk/remote.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package chunk
-
-import (
- "context"
- "io"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type RemoteChunk struct {
- chunkId string
- client api.ChunkStorageClient
-}
-
-func (r *RemoteChunk) Stats() (info ChunkInfo, err error) {
- resp, err := r.client.GetChunkStatus(
- context.Background(),
- &api.GetChunkStatusRequest{ChunkId: r.chunkId})
- if err != nil {
- return
- }
- info = ChunkInfo{
- resp.Status,
- int(resp.TotalBytes),
- int(resp.CommittedBytes)}
- return
-}
-
-func (r *RemoteChunk) ReaderAt() io.ReaderAt {
- return &remoteChunkReaderAt{
- chunkId: r.chunkId,
- client: r.client}
-}
-
-func (r *RemoteChunk) WriterAt() io.WriterAt {
- return &remoteChunkWriterAt{
- chunkId: r.chunkId,
- client: r.client}
-}
-
-type remoteChunkReaderAt struct {
- chunkId string
- client api.ChunkStorageClient
-}
-
-func (c *remoteChunkReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
- req := api.ReadChunkRequest{
- ChunkId: c.chunkId,
- Offset: int32(offset),
- NumBytes: int32(len(p))}
- resp, err := c.client.ReadChunk(context.Background(), &req)
- if err != nil {
- return
- }
- n = copy(p, resp.Data)
- return
-}
-
-type remoteChunkWriterAt struct {
- chunkId string
- client api.ChunkStorageClient
-}
-
-func (c *remoteChunkWriterAt) WriteAt(p []byte, offset int64) (n int, err error) {
- req := api.WriteChunkRequest{
- ChunkId: c.chunkId,
- Offset: int32(offset),
- Data: p}
- resp, err := c.client.WriteChunk(context.Background(), &req)
- if resp != nil {
- n = int(resp.BytesWritten)
- }
- return
-}
diff --git a/archive/pfs/chunk/replicator.go b/archive/pfs/chunk/replicator.go
deleted file mode 100644
index d8990a7..0000000
--- a/archive/pfs/chunk/replicator.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package chunk
-
-import (
- "context"
- "io"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type ReplicaAssignmentChangeListener interface {
- Primary(chunkId, currentPrimary string) <-chan string
-}
-
-type PrimaryReplicaChangeListener interface {
- ChunkId() string
- Address() <-chan string
-}
-
-type NonChangingReplicaAssignment struct {
-}
-
-func (l *NonChangingReplicaAssignment) Primary(chunkId, address string) <-chan string {
- ch := make(chan string, 1)
- ch <- address
- return ch
-}
-
-func replicate(ctx context.Context, dst, src Chunk, done chan<- int) {
- dstInfo, err := dst.Stats()
- if err != nil {
- panic(err)
- }
- inp := src.ReaderAt()
- replicated := dstInfo.Committed
- out := dst.WriterAt()
- for {
- select {
- default:
- p := make([]byte, 100)
- n, err := inp.ReadAt(p, int64(replicated))
- if n > 0 {
- m, _ := out.WriteAt(p[:n], int64(replicated))
- replicated += m
- }
- if err == io.EOF {
- done <- 1
- return
- }
-
- case <-ctx.Done():
- return
- }
- }
-}
-
-func ReplicateFromPrimary(ctx context.Context, chunkId string, dst Chunk, primaryAddressCh <-chan string) {
- var done chan int
- var cancel context.CancelFunc = nil
- for {
- select {
- case <-done:
- return
- case <-ctx.Done():
- return
- case address := <-primaryAddressCh:
- if cancel != nil {
- cancel()
- }
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial(address, opts...)
- if err == nil {
- continue
- }
- client := api.NewChunkStorageClient(conn)
- src := RemoteChunk{chunkId, client}
- replicatorCtx, cancelFn := context.WithCancel(context.Background())
- cancel = cancelFn
- done = make(chan int, 1)
- go replicate(replicatorCtx, dst, &src, done)
- }
- }
-}
-
-func WriteToPrimary(ctx context.Context, chunkId string, src Chunk, primaryAddressCh <-chan string) {
- var done chan int
- var cancel context.CancelFunc = nil
- for {
- select {
- case <-done:
- return
- case <-ctx.Done():
- return
- case address := <-primaryAddressCh:
- if cancel != nil {
- cancel()
- }
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial(address, opts...)
- if err != nil {
- continue
- }
- client := api.NewChunkStorageClient(conn)
- dst := RemoteChunk{chunkId, client}
- replicatorCtx, cancelFn := context.WithCancel(context.Background())
- cancel = cancelFn
- done = make(chan int, 1)
- go replicate(replicatorCtx, &dst, src, done)
- }
- }
-}
diff --git a/archive/pfs/chunk/server.go b/archive/pfs/chunk/server.go
deleted file mode 100644
index d619f13..0000000
--- a/archive/pfs/chunk/server.go
+++ /dev/null
@@ -1,123 +0,0 @@
-package chunk
-
-import (
- "context"
- "fmt"
- "log"
-
- "sync"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type ChunkServer struct {
- factory ChunkFactory
- assignmentChangeLis ReplicaAssignmentChangeListener
- chunks sync.Map
- replicatorCancel sync.Map
-}
-
-func NewChunkServer(factory ChunkFactory,
- assignmentChangeLis ReplicaAssignmentChangeListener) *ChunkServer {
- return &ChunkServer{
- factory: factory,
- assignmentChangeLis: assignmentChangeLis}
-}
-
-func (s *ChunkServer) ListChunks(
- ctx context.Context,
- req *api.ListChunksRequest) (resp *api.ListChunksResponse, err error) {
- resp = &api.ListChunksResponse{}
- s.chunks.Range(func(k, v interface{}) bool {
- resp.ChunkId = append(resp.ChunkId, k.(string))
- return true
- })
- return
-}
-
-func (s *ChunkServer) CreateChunk(
- ctx context.Context,
- req *api.CreateChunkRequest) (resp *api.CreateChunkResponse, err error) {
- chunk := s.factory.New(int(req.Size))
- s.chunks.Store(req.ChunkId, chunk)
- switch req.Role {
- case api.ReplicaRole_SECONDARY:
- ctx, cancel := context.WithCancel(context.Background())
- s.replicatorCancel.Store(req.ChunkId, cancel)
- primaryAddressCh := s.assignmentChangeLis.Primary(
- req.ChunkId, req.PrimaryAddress)
- go ReplicateFromPrimary(ctx, req.ChunkId, chunk, primaryAddressCh)
- case api.ReplicaRole_PRIMARY:
- {
- }
- }
- resp = &api.CreateChunkResponse{}
- log.Printf("Created chunk: %s\n", req.ChunkId)
- return
-
-}
-
-func (s *ChunkServer) GetChunkStatus(
- ctx context.Context,
- req *api.GetChunkStatusRequest) (resp *api.GetChunkStatusResponse, err error) {
- if chunk, ok := s.chunks.Load(req.ChunkId); ok {
- c := chunk.(Chunk)
- var info ChunkInfo
- info, err = c.Stats()
- if err != nil {
- return
- }
- resp = &api.GetChunkStatusResponse{
- Status: info.Status,
- TotalBytes: int32(info.Size),
- CommittedBytes: int32(info.Committed)}
- return
- }
- return nil, fmt.Errorf("Could not fund chunk: %s", req.ChunkId)
-}
-
-func (s *ChunkServer) ReadChunk(
- ctx context.Context,
- req *api.ReadChunkRequest) (resp *api.ReadChunkResponse, err error) {
- if value, ok := s.chunks.Load(req.ChunkId); ok {
- chunk := value.(Chunk)
- b := make([]byte, req.NumBytes)
- var n int
- n, err = chunk.ReaderAt().ReadAt(b, int64(req.Offset))
- if n == 0 {
- return
- }
- return &api.ReadChunkResponse{Data: b[:n]}, nil
-
- } else {
- return nil, fmt.Errorf("Chunk not found: %s", req.ChunkId)
- }
-}
-
-func (s *ChunkServer) WriteChunk(
- ctx context.Context,
- req *api.WriteChunkRequest) (resp *api.WriteChunkResponse, err error) {
- if value, ok := s.chunks.Load(req.ChunkId); ok {
- chunk := value.(Chunk)
- var n int
- n, err = chunk.WriterAt().WriteAt(req.Data, int64(req.Offset))
- if n == 0 {
- return
- }
- return &api.WriteChunkResponse{BytesWritten: int32(n)}, nil
-
- } else {
- return nil, fmt.Errorf("Chunk not found: %s", req.ChunkId)
- }
-}
-
-func (s *ChunkServer) RemoveChunk(
- ctx context.Context,
- req *api.RemoveChunkRequest) (resp *api.RemoveChunkResponse, err error) {
- if cancel, ok := s.replicatorCancel.Load(req.ChunkId); ok {
- cancel.(context.CancelFunc)()
- s.replicatorCancel.Delete(req.ChunkId)
- }
- s.chunks.Delete(req.ChunkId)
- return &api.RemoveChunkResponse{}, nil
-}
diff --git a/archive/pfs/chunk/server_test.go b/archive/pfs/chunk/server_test.go
deleted file mode 100644
index 9549a06..0000000
--- a/archive/pfs/chunk/server_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package chunk
-
-import (
- "bytes"
- "context"
- "testing"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-func TestStoreChunk(t *testing.T) {
- s := ChunkServer{factory: &InMemoryChunkFactory{}}
- _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
- ChunkId: "foo",
- Size: 11,
- Role: api.ReplicaRole_PRIMARY})
- if err != nil {
- t.Error(err)
- }
- _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
- ChunkId: "foo",
- Offset: 0,
- Data: []byte("hello world")})
- if err != nil {
- t.Error(err)
- }
-}
-
-func TestStoreAndReadChunk(t *testing.T) {
- s := ChunkServer{factory: &InMemoryChunkFactory{}}
- _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
- ChunkId: "foo",
- Size: 11,
- Role: api.ReplicaRole_PRIMARY})
- if err != nil {
- t.Error(err)
- }
- _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
- ChunkId: "foo",
- Offset: 0,
- Data: []byte("hello world")})
- if err != nil {
- t.Error(err)
- }
- resp, err := s.ReadChunk(context.Background(), &api.ReadChunkRequest{
- ChunkId: "foo",
- NumBytes: 100})
- if err != nil {
- t.Error(err)
- }
- if bytes.Compare(resp.Data, []byte("hello world")) != 0 {
- t.Errorf("Expected: %s\nGot: %s\n", "hello world", resp.Data)
- }
-}
-
-func TestReadWithOffsets(t *testing.T) {
- s := ChunkServer{factory: &InMemoryChunkFactory{}}
- _, err := s.CreateChunk(context.Background(), &api.CreateChunkRequest{
- ChunkId: "foo",
- Size: 11,
- Role: api.ReplicaRole_PRIMARY})
- if err != nil {
- t.Error(err)
- }
- _, err = s.WriteChunk(context.Background(), &api.WriteChunkRequest{
- ChunkId: "foo",
- Offset: 0,
- Data: []byte("hello world")})
- if err != nil {
- t.Error(err)
- }
- resp, err := s.ReadChunk(context.Background(), &api.ReadChunkRequest{
- ChunkId: "foo",
- Offset: 0,
- NumBytes: 2})
- if err != nil {
- t.Error(err)
- }
- if bytes.Compare(resp.Data, []byte("he")) != 0 {
- t.Errorf("Expected: %s\nGot: %s\n", "he", resp.Data)
- }
- resp, err = s.ReadChunk(context.Background(), &api.ReadChunkRequest{
- ChunkId: "foo",
- Offset: 2,
- NumBytes: 2})
- if err != nil {
- t.Error(err)
- }
- if bytes.Compare(resp.Data, []byte("ll")) != 0 {
- t.Errorf("Expected: %s\nGot: %s\n", "ll", resp.Data)
- }
- resp, err = s.ReadChunk(context.Background(), &api.ReadChunkRequest{
- ChunkId: "foo",
- Offset: 4,
- NumBytes: 100})
- if err != nil {
- t.Error(err)
- }
- if bytes.Compare(resp.Data, []byte("o world")) != 0 {
- t.Errorf("Expected: %s\nGot: %s\n", "o world", resp.Data)
- }
-
-}
diff --git a/archive/pfs/chunk_server.go b/archive/pfs/chunk_server.go
deleted file mode 100644
index 04a266e..0000000
--- a/archive/pfs/chunk_server.go
+++ /dev/null
@@ -1,55 +0,0 @@
-package main
-
-import (
- "context"
- "flag"
- "fmt"
- "log"
- "net"
- "time"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
- "github.com/giolekva/pcloud/pfs/chunk"
-)
-
-var controllerAddress = flag.String("controller", "localhost:123", "Metadata storage address.")
-var selfAddress = flag.String("self", "", "Metadata storage address.")
-
-func main() {
- flag.Parse()
- log.Print("Chunk server starting")
-
- // Create Master server client.
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial(*controllerAddress, opts...)
- if err != nil {
- log.Fatalf("Failed to dial %s: %v", *controllerAddress, err)
- }
- defer conn.Close()
- client := api.NewMetadataStorageClient(conn)
-
- // Register current Chunk server with Master.
- ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
- _, err = client.AddChunkServer(
- ctx,
- &api.AddChunkServerRequest{Address: *selfAddress})
- if err != nil {
- log.Fatalf("failed to register chunk server: %v", err)
- }
- log.Print("Registered myself")
-
- // Start RPC server
- lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 234))
- if err != nil {
- log.Fatalf("failed to listen: %v", err)
- }
- server := grpc.NewServer()
- api.RegisterChunkStorageServer(server, chunk.NewChunkServer(
- &chunk.InMemoryChunkFactory{},
- &chunk.NonChangingReplicaAssignment{}))
- server.Serve(lis)
-}
diff --git a/archive/pfs/client.go b/archive/pfs/client.go
deleted file mode 100644
index 2abb47f..0000000
--- a/archive/pfs/client.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package main
-
-import (
- "flag"
- "log"
- "os"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
- "github.com/giolekva/pcloud/pfs/client"
-)
-
-var controllerAddress = flag.String("controller", "localhost:123", "Metadata storage address.")
-var fileToUpload = flag.String("file", "", "File path to upload.")
-
-func main() {
- flag.Parse()
-
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial(*controllerAddress, opts...)
- if err != nil {
- log.Fatalf("Failed to dial %s: %v", *controllerAddress, err)
- }
- defer conn.Close()
- uploader := client.NewFileUploader(api.NewMetadataStorageClient(conn))
-
- f, err := os.Open(*fileToUpload)
- if err != nil {
- panic(err)
- }
-
- uploader.Upload(f)
-}
diff --git a/archive/pfs/client/client.go b/archive/pfs/client/client.go
deleted file mode 100644
index a8492a3..0000000
--- a/archive/pfs/client/client.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package client
-
-import (
- "context"
- "os"
-
- "github.com/giolekva/pcloud/pfs/api"
- "github.com/giolekva/pcloud/pfs/chunk"
-)
-
-type FileUploader struct {
- client api.MetadataStorageClient
-}
-
-func NewFileUploader(client api.MetadataStorageClient) *FileUploader {
- return &FileUploader{client}
-}
-
-func (fu *FileUploader) Upload(f *os.File, numReplicas int) {
- info, err := f.Stat()
- if err != nil {
- return
- }
- resp, err := fu.client.CreateBlob(
- context.Background(), &api.CreateBlobRequest{
- SizeBytes: int32(info.Size()),
- NumReplicas: int32(numReplicas)})
- if err != nil {
- panic(err)
- }
- if len(resp.Chunk) != 1 {
- panic(resp)
- }
- lis := &chunk.NonChangingReplicaAssignment{}
- primaryAddressCh := lis.Primary(
- resp.Chunk[0].ChunkId,
- resp.Chunk[0].Server[0])
- chunk.WriteToPrimary(
- context.Background(),
- resp.Chunk[0].ChunkId,
- chunk.NewReadOnlyFileChunk(f, 0, int(info.Size())),
- primaryAddressCh)
-}
diff --git a/archive/pfs/client/client_test.go b/archive/pfs/client/client_test.go
deleted file mode 100644
index b9f3002..0000000
--- a/archive/pfs/client/client_test.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package client
-
-import (
- "os"
- "testing"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
- pt "github.com/giolekva/pcloud/pfs/testing"
-)
-
-func TestUploadSmallFile(t *testing.T) {
- env, err := pt.NewInMemoryEnv(1)
- if err != nil {
- t.Error(err)
- }
- defer env.Stop()
-
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial("unix:///tmp/pcloud/controller", opts...)
- if err != nil {
- t.Error(err)
- }
- defer conn.Close()
- client := api.NewMetadataStorageClient(conn)
-
- uploader := NewFileUploader(client)
- f, err := os.Open("testdata/foo")
- if err != nil {
- t.Error(err)
- }
- uploader.Upload(f, 1)
-
-}
-
-func TestUploadSmallFileWithReplication(t *testing.T) {
- env, err := pt.NewInMemoryEnv(3)
- if err != nil {
- t.Error(err)
- }
- defer env.Stop()
-
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial("unix:///tmp/pcloud/controller", opts...)
- if err != nil {
- t.Error(err)
- }
- defer conn.Close()
- client := api.NewMetadataStorageClient(conn)
-
- uploader := NewFileUploader(client)
- f, err := os.Open("testdata/foo")
- if err != nil {
- t.Error(err)
- }
- uploader.Upload(f, 2)
-}
diff --git a/archive/pfs/client/testdata/foo b/archive/pfs/client/testdata/foo
deleted file mode 100644
index 257cc56..0000000
--- a/archive/pfs/client/testdata/foo
+++ /dev/null
@@ -1 +0,0 @@
-foo
diff --git a/archive/pfs/controller/chunk.go b/archive/pfs/controller/chunk.go
deleted file mode 100644
index 5fb694f..0000000
--- a/archive/pfs/controller/chunk.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package controller
-
-import (
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type chunkServerStatus int
-
-const (
- Healthy chunkServerStatus = iota
- UNREACHABLE
-)
-
-type chunkServer struct {
- address string
- status chunkServerStatus
- chunks map[string]api.ChunkStatus
-}
diff --git a/archive/pfs/controller/server.go b/archive/pfs/controller/server.go
deleted file mode 100644
index 885ad55..0000000
--- a/archive/pfs/controller/server.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package controller
-
-import (
- "context"
- "log"
- "math/rand"
-
- "github.com/google/uuid"
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
-)
-
-type chunkServers struct {
- address string
-}
-
-type BlobStatus int
-
-const (
- NEW BlobStatus = iota
-)
-
-type ChunkStatus int
-
-const (
- ASSIGNED ChunkStatus = iota
- STORED
-)
-
-type chunkReplica struct {
- chunkServer string
- status ChunkStatus
-}
-
-type chunk struct {
- id string
- replica []chunkReplica
-}
-
-type blob struct {
- id string
- status BlobStatus
- chunks []chunk
-}
-
-type MasterServer struct {
- chunkServers []*chunkServer
- blobs []*blob
-}
-
-func NewMasterServer() *MasterServer {
- return &MasterServer{}
-}
-
-func (s *MasterServer) AddChunkServer(
- ctx context.Context,
- req *api.AddChunkServerRequest) (*api.AddChunkServerResponse, error) {
- s.chunkServers = append(s.chunkServers, &chunkServer{
- address: req.Address,
- status: Healthy})
- log.Printf("Registered Chunk server: %s", req.Address)
- return &api.AddChunkServerResponse{}, nil
-}
-
-func (s *MasterServer) CreateBlob(
- ctx context.Context,
- req *api.CreateBlobRequest) (*api.CreateBlobResponse, error) {
- if int(req.NumReplicas) > len(s.chunkServers) {
- return nil, nil
- }
- resp := api.CreateBlobResponse{
- BlobId: uuid.New().String(),
- Chunk: []*api.ChunkStorageMetadata{
- {ChunkId: uuid.New().String()},
- }}
- assigned := 0
- chunkId := resp.Chunk[0].ChunkId
- for i := range rand.Perm(len(s.chunkServers)) {
- if assigned == int(req.NumReplicas) {
- break
- }
- address := s.chunkServers[i].address
- log.Printf(address)
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial(address, opts...)
- if err != nil {
- continue
- }
- defer conn.Close()
- client := api.NewChunkStorageClient(conn)
- createChunkReq := api.CreateChunkRequest{
- ChunkId: chunkId,
- Size: req.SizeBytes}
- if assigned == 0 {
- createChunkReq.Role = api.ReplicaRole_PRIMARY
- } else {
- createChunkReq.Role = api.ReplicaRole_SECONDARY
- createChunkReq.PrimaryAddress = resp.Chunk[0].Server[0]
- }
- _, err = client.CreateChunk(ctx, &createChunkReq)
- if err == nil {
- resp.Chunk[0].Server = append(resp.Chunk[0].Server, address)
- assigned++
- }
- }
- return &resp, nil
-}
-
-func (s *MasterServer) GetBlobMetadata(
- ctx context.Context,
- req *api.GetBlobMetadataRequest) (*api.GetBlobMetadataResponse, error) {
- return nil, nil
-}
diff --git a/archive/pfs/controller_server.go b/archive/pfs/controller_server.go
deleted file mode 100644
index c905ba3..0000000
--- a/archive/pfs/controller_server.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "log"
- "net"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
- "github.com/giolekva/pcloud/pfs/controller"
-)
-
-var port = flag.Int("port", 123, "Port to listen on.")
-
-func main() {
- flag.Parse()
- log.Print("Master server starting")
- lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
- if err != nil {
- log.Fatalf("Failed to listen on port %d: %v", *port, err)
- }
- log.Printf("Listening on port: %d", *port)
- server := grpc.NewServer()
- api.RegisterMetadataStorageServer(server, controller.NewMasterServer())
- log.Print("Master serving")
- server.Serve(lis)
-}
diff --git a/archive/pfs/pfs.yamls b/archive/pfs/pfs.yamls
deleted file mode 100644
index ba06bcb..0000000
--- a/archive/pfs/pfs.yamls
+++ /dev/null
@@ -1,78 +0,0 @@
----
-kind: Service
-apiVersion: v1
-metadata:
- name: pfs-controller-service
-spec:
- type: ClusterIP
- selector:
- app: pfs-controller
- ports:
- - nodePort:
- port: 111
- targetPort: 123
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
- name: pfs-controller
-spec:
- selector:
- matchLabels:
- app: pfs-controller
- replicas: 1
- template:
- metadata:
- labels:
- app: pfs-controller
- spec:
- containers:
- - name: pfs-controller
- image: pcloud:latest
- imagePullPolicy: Never
- ports:
- - containerPort: 123
- volumeMounts:
- - name: code
- mountPath: /src/go/src/github.com/giolekva/pcloud/pfs
- command: ["/bin/sh"]
- args: ["-c", "protoc api/api.proto --go_out=plugins=grpc:. && go run controller_server.go --port=123"]
- volumes:
- - name: code
- hostPath:
- path: "/Users/lekva/dev/go/src/github.com/giolekva/pcloud/pfs"
----
-kind: Deployment
-apiVersion: apps/v1
-metadata:
- name: pfs-chunk
-spec:
- selector:
- matchLabels:
- app: pfs-chunk
- replicas: 3
- template:
- metadata:
- labels:
- app: pfs-chunk
- spec:
- containers:
- - name: pfs-chunk
- image: pcloud:latest
- imagePullPolicy: Never
- ports:
- - containerPort: 234
- env:
- - name: SELF_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- volumeMounts:
- - name: code
- mountPath: /src/go/src/github.com/giolekva/pcloud/pfs
- command: ["/bin/sh"]
- args: ["-c", "protoc api/api.proto --go_out=plugins=grpc:. && go run chunk_server.go --controller=pfs-controller-service:111 --self=$(SELF_IP):234"]
- volumes:
- - name: code
- hostPath:
- path: "/Users/lekva/dev/go/src/github.com/giolekva/pcloud/pfs"
diff --git a/archive/pfs/testing/in_memory_env.go b/archive/pfs/testing/in_memory_env.go
deleted file mode 100644
index c3410be..0000000
--- a/archive/pfs/testing/in_memory_env.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package testing
-
-import (
- "context"
- "fmt"
- "net"
- "syscall"
- "time"
-
- "google.golang.org/grpc"
-
- "github.com/giolekva/pcloud/pfs/api"
- "github.com/giolekva/pcloud/pfs/chunk"
- "github.com/giolekva/pcloud/pfs/controller"
-)
-
-type InMemoryEnv struct {
- m *grpc.Server
- c []*grpc.Server
- controllerConn *grpc.ClientConn
-}
-
-func NewInMemoryEnv(numChunkServers int) (*InMemoryEnv, error) {
- env := new(InMemoryEnv)
- syscall.Unlink("/tmp/pcloud/controller")
- lis, err := net.Listen("unix", "/tmp/pcloud/controller")
- if err != nil {
- return nil, err
- }
- server := grpc.NewServer()
- api.RegisterMetadataStorageServer(server, controller.NewMasterServer())
- go server.Serve(lis)
-
- var opts []grpc.DialOption
- opts = append(opts, grpc.WithInsecure())
- opts = append(opts, grpc.WithBlock())
- conn, err := grpc.Dial("unix:/tmp/pcloud/controller", opts...)
- if err != nil {
- return nil, err
- }
- env.controllerConn = conn
- client := api.NewMetadataStorageClient(conn)
-
- env.c = make([]*grpc.Server, numChunkServers)
- for i := 0; i < numChunkServers; i++ {
- unixSocket := fmt.Sprintf("/tmp/pcloud/chunk-%d", i)
- syscall.Unlink(unixSocket)
- lis, err := net.Listen("unix", unixSocket)
- if err != nil {
- return nil, err
- }
- server := grpc.NewServer()
- api.RegisterChunkStorageServer(server, chunk.NewChunkServer(
- &chunk.InMemoryChunkFactory{},
- &chunk.NonChangingReplicaAssignment{}))
- go server.Serve(lis)
- env.c[i] = server
- }
-
- for i := 0; i < numChunkServers; i++ {
- ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
- _, err = client.AddChunkServer(
- ctx,
- &api.AddChunkServerRequest{Address: fmt.Sprintf("unix:///tmp/pcloud/chunk-%d", i)})
- if err != nil {
- return nil, err
- }
- }
- return env, nil
-}
-
-func (e *InMemoryEnv) Stop() {
- if e.controllerConn != nil {
- e.controllerConn.Close()
- }
- for _, s := range e.c {
- if s != nil {
- s.GracefulStop()
- }
- }
- if e.m != nil {
- e.m.GracefulStop()
- }
-}
diff --git a/archive/pfs/testing/simple_test.go b/archive/pfs/testing/simple_test.go
deleted file mode 100644
index d8e2510..0000000
--- a/archive/pfs/testing/simple_test.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package testing
-
-import (
- "testing"
-)
-
-func TestSetup(t *testing.T) {
- env, err := NewInMemoryEnv(3)
- if err != nil {
- t.Error(err)
- }
- defer env.Stop()
-}
diff --git a/bazel_tools/docker.bzl b/bazel_tools/docker.bzl
index 1149497..b45c72c 100644
--- a/bazel_tools/docker.bzl
+++ b/bazel_tools/docker.bzl
@@ -32,24 +32,25 @@
docker push \$$IMAGE
"""
-
def docker_image(name, registry, image, tag, dockerfile, srcs, **kwargs):
native.genrule(
- name = "%s.sh" % name,
- executable = False,
- outs = ["build_and_push.sh",],
- cmd = """cat > $@ <<EOM
+ name = "%s.sh" % name,
+ executable = False,
+ outs = ["build_and_push.sh"],
+ cmd = """cat > $@ <<EOM
%s
EOM
""" % __PUSH_TO_DEV_TMPL.format(
- registry = registry,
- image = image,
- tag = tag,
- dockerfile = dockerfile,
- package = native.package_name(),
- ))
+ registry = registry,
+ image = image,
+ tag = tag,
+ dockerfile = dockerfile,
+ package = native.package_name(),
+ ),
+ )
native.sh_binary(
- name = name,
- srcs = ["build_and_push.sh",],
- data = srcs + [dockerfile,],
- deps = ["@bazel_tools//tools/bash/runfiles",])
+ name = name,
+ srcs = ["build_and_push.sh"],
+ data = srcs + [dockerfile],
+ deps = ["@bazel_tools//tools/bash/runfiles"],
+ )
diff --git a/bazel_tools/helm.bzl b/bazel_tools/helm.bzl
index 7b71492..8635257 100644
--- a/bazel_tools/helm.bzl
+++ b/bazel_tools/helm.bzl
@@ -26,31 +26,55 @@
helm --namespace={namespace} install --create-namespace {release_name} \$$CHART_TARBALL {args}
"""
+__HELM_UNINSTALL_TMPL = """
+helm --namespace={namespace} uninstall {release_name}
+"""
+
def helm_install(name, namespace, release_name, chart, args):
args_str = ""
for arg, value in args.items():
args_str += "--set %s=%s " % (arg, value)
native.genrule(
- name = "%s.sh" % name,
- executable = False,
- srcs = [chart,],
- outs = ["helm_install.sh",],
- cmd = """cat > $@ <<EOM
+ name = "%s.sh" % name,
+ executable = False,
+ srcs = [chart],
+ outs = ["helm_install.sh"],
+ cmd = """cat > $@ <<EOM
%s
EOM
""" % __HELM_INSTALL_TMPL.format(
- namespace = namespace,
- release_name = release_name,
- package = native.package_name(),
- chart = "%s.tar.gz" % chart.split(":")[1],
- args = args_str,
- ))
+ namespace = namespace,
+ release_name = release_name,
+ package = native.package_name(),
+ chart = "%s.tar.gz" % chart.split(":")[1],
+ args = args_str,
+ ),
+ )
native.sh_binary(
- name = name,
- srcs = ["helm_install.sh",],
- data = [
- chart,
- ],
- deps = [
- "@bazel_tools//tools/bash/runfiles",
- ])
+ name = name,
+ srcs = ["helm_install.sh"],
+ data = [
+ chart,
+ ],
+ deps = [
+ "@bazel_tools//tools/bash/runfiles",
+ ],
+ )
+
+def helm_uninstall(name, namespace, release_name):
+ native.genrule(
+ name = "%s.sh" % name,
+ executable = False,
+ outs = ["helm_uninstall.sh"],
+ cmd = """cat > $@ <<EOM
+%s
+EOM
+""" % __HELM_UNINSTALL_TMPL.format(
+ namespace = namespace,
+ release_name = release_name,
+ ),
+ )
+ native.sh_binary(
+ name = name,
+ srcs = ["helm_uninstall.sh"],
+ )
diff --git a/controller/BUILD b/controller/BUILD
deleted file mode 100644
index 48e6b37..0000000
--- a/controller/BUILD
+++ /dev/null
@@ -1,32 +0,0 @@
-load("//:bazel_tools/docker.bzl", "docker_image")
-load("//:bazel_tools/helm.bzl", "helm_install")
-load("@rules_pkg//:pkg.bzl", "pkg_tar")
-
-# TODO(lekva): figure out how to build py_binary with pip dependencies and
-# migrate off docker_image rule
-docker_image(
- name = "push_to_dev",
- registry = "localhost:30500",
- image = "giolekva/pcloud-api-server",
- tag = "latest",
- dockerfile = "Dockerfile",
- srcs = glob(["**"], exclude=["Dockerfile"]),
-)
-
-pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
-)
-
-helm_install(
- name = "install",
- namespace = "pcloud",
- release_name = "init",
- chart = ":chart",
- args = {
- "image.name": "localhost:30500/giolekva/pcloud-api-server",
- "image.pullPolicy": "Always",
- },
-)
diff --git a/core/api/BUILD b/core/api/BUILD
new file mode 100644
index 0000000..ff865ca
--- /dev/null
+++ b/core/api/BUILD
@@ -0,0 +1,41 @@
+load("//:bazel_tools/docker.bzl", "docker_image")
+load("//:bazel_tools/helm.bzl", "helm_install", "helm_uninstall")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+
+# TODO(lekva): figure out how to build py_binary with pip dependencies and
+# migrate off docker_image rule
+docker_image(
+ name = "push_to_dev",
+ srcs = glob(
+ ["**"],
+ exclude = ["Dockerfile"],
+ ),
+ dockerfile = "Dockerfile",
+ image = "giolekva/pcloud-api-server",
+ registry = "localhost:30500",
+ tag = "latest",
+)
+
+pkg_tar(
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
+)
+
+helm_install(
+ name = "install",
+ args = {
+ "image.name": "localhost:30500/giolekva/pcloud-api-server",
+ "image.pullPolicy": "Always",
+ },
+ chart = ":chart",
+ namespace = "pcloud",
+ release_name = "init",
+)
+
+helm_uninstall(
+ name = "uninstall",
+ namespace = "pcloud",
+ release_name = "init",
+)
diff --git a/controller/Dockerfile b/core/api/Dockerfile
similarity index 93%
rename from controller/Dockerfile
rename to core/api/Dockerfile
index a9fabd8..0d7f980 100644
--- a/controller/Dockerfile
+++ b/core/api/Dockerfile
@@ -9,7 +9,7 @@
# RUN rm protoc-3.11.4-linux-x86_64.zip
# ENV PATH=/protoc/bin:$PATH
-WORKDIR $GOPATH/src/github.com/giolekva/pcloud/controller
+WORKDIR $GOPATH/src/github.com/giolekva/pcloud/core/api
COPY . .
# RUN go get -v ./...
diff --git a/controller/chart/Chart.yaml b/core/api/chart/Chart.yaml
similarity index 100%
rename from controller/chart/Chart.yaml
rename to core/api/chart/Chart.yaml
diff --git a/controller/chart/README.md b/core/api/chart/README.md
similarity index 100%
rename from controller/chart/README.md
rename to core/api/chart/README.md
diff --git a/controller/chart/templates/deployment.yaml b/core/api/chart/templates/deployment.yaml
similarity index 100%
rename from controller/chart/templates/deployment.yaml
rename to core/api/chart/templates/deployment.yaml
diff --git a/controller/chart/templates/service.yaml b/core/api/chart/templates/service.yaml
similarity index 100%
rename from controller/chart/templates/service.yaml
rename to core/api/chart/templates/service.yaml
diff --git a/controller/chart/values.yaml b/core/api/chart/values.yaml
similarity index 100%
rename from controller/chart/values.yaml
rename to core/api/chart/values.yaml
diff --git a/controller/go.mod b/core/api/go.mod
similarity index 86%
rename from controller/go.mod
rename to core/api/go.mod
index c372a57..b163bb3 100644
--- a/controller/go.mod
+++ b/core/api/go.mod
@@ -1,4 +1,4 @@
-module github.com/giolekva/pcloud/controller
+module github.com/giolekva/pcloud/core/api
go 1.14
diff --git a/controller/go.sum b/core/api/go.sum
similarity index 100%
rename from controller/go.sum
rename to core/api/go.sum
diff --git a/controller/main.go b/core/api/main.go
similarity index 97%
rename from controller/main.go
rename to core/api/main.go
index 5c3df86..db4f841 100644
--- a/controller/main.go
+++ b/core/api/main.go
@@ -9,7 +9,7 @@
"log"
"net/http"
- "github.com/giolekva/pcloud/controller/schema"
+ "github.com/giolekva/pcloud/core/api/schema"
"github.com/golang/glog"
"github.com/itaysk/regogo"
diff --git a/controller/schema/dgraph_schema_store.go b/core/api/schema/dgraph_schema_store.go
similarity index 100%
rename from controller/schema/dgraph_schema_store.go
rename to core/api/schema/dgraph_schema_store.go
diff --git a/controller/schema/schema.go b/core/api/schema/schema.go
similarity index 100%
rename from controller/schema/schema.go
rename to core/api/schema/schema.go
diff --git a/controller/tests/query_test.go b/core/api/tests/query_test.go
similarity index 100%
rename from controller/tests/query_test.go
rename to core/api/tests/query_test.go
diff --git a/core/appmanager/BUILD b/core/appmanager/BUILD
new file mode 100644
index 0000000..1cc903a
--- /dev/null
+++ b/core/appmanager/BUILD
@@ -0,0 +1,41 @@
+load("//:bazel_tools/docker.bzl", "docker_image")
+load("//:bazel_tools/helm.bzl", "helm_install", "helm_uninstall")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+
+# TODO(lekva): figure out how to build py_binary with pip dependencies and
+# migrate off docker_image rule
+docker_image(
+ name = "push_to_dev",
+ srcs = glob(
+ ["**"],
+ exclude = ["Dockerfile"],
+ ),
+ dockerfile = "Dockerfile",
+ image = "giolekva/pcloud-app-manager",
+ registry = "localhost:30500",
+ tag = "latest",
+)
+
+pkg_tar(
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
+)
+
+helm_install(
+ name = "install",
+ args = {
+ "image.name": "localhost:30500/giolekva/pcloud-app-manager",
+ "image.pullPolicy": "Always",
+ },
+ chart = ":chart",
+ namespace = "pcloud-app-manager",
+ release_name = "init",
+)
+
+helm_uninstall(
+ name = "uninstall",
+ namespace = "pcloud-app-manager",
+ release_name = "init",
+)
diff --git a/appmanager/Dockerfile b/core/appmanager/Dockerfile
similarity index 89%
rename from appmanager/Dockerfile
rename to core/appmanager/Dockerfile
index 248c052..791ab67 100644
--- a/appmanager/Dockerfile
+++ b/core/appmanager/Dockerfile
@@ -12,7 +12,7 @@
RUN wget -O helm.tar.gz https://get.helm.sh/helm-v3.2.1-$GOOS-$GOARCH.tar.gz
RUN tar -xvf helm.tar.gz
-WORKDIR $GOPATH/src/github.com/giolekva/pcloud/appmanager
+WORKDIR $GOPATH/src/github.com/giolekva/pcloud/core/appmanager
COPY . .
RUN go build -o $GOPATH/bin/app-manager -trimpath -ldflags="-s -w" cmd/main.go
diff --git a/appmanager/actions.go b/core/appmanager/actions.go
similarity index 100%
rename from appmanager/actions.go
rename to core/appmanager/actions.go
diff --git a/appmanager/chart/Chart.yaml b/core/appmanager/chart/Chart.yaml
similarity index 100%
rename from appmanager/chart/Chart.yaml
rename to core/appmanager/chart/Chart.yaml
diff --git a/appmanager/chart/README.md b/core/appmanager/chart/README.md
similarity index 100%
rename from appmanager/chart/README.md
rename to core/appmanager/chart/README.md
diff --git a/appmanager/chart/templates/cluster-role-binding.yaml b/core/appmanager/chart/templates/cluster-role-binding.yaml
similarity index 100%
rename from appmanager/chart/templates/cluster-role-binding.yaml
rename to core/appmanager/chart/templates/cluster-role-binding.yaml
diff --git a/appmanager/chart/templates/cluster-role.yaml b/core/appmanager/chart/templates/cluster-role.yaml
similarity index 100%
rename from appmanager/chart/templates/cluster-role.yaml
rename to core/appmanager/chart/templates/cluster-role.yaml
diff --git a/appmanager/chart/templates/ingress.yaml b/core/appmanager/chart/templates/ingress.yaml
similarity index 100%
rename from appmanager/chart/templates/ingress.yaml
rename to core/appmanager/chart/templates/ingress.yaml
diff --git a/appmanager/chart/templates/service.yaml b/core/appmanager/chart/templates/service.yaml
similarity index 100%
rename from appmanager/chart/templates/service.yaml
rename to core/appmanager/chart/templates/service.yaml
diff --git a/appmanager/chart/templates/statefulset.yaml b/core/appmanager/chart/templates/statefulset.yaml
similarity index 100%
rename from appmanager/chart/templates/statefulset.yaml
rename to core/appmanager/chart/templates/statefulset.yaml
diff --git a/appmanager/chart/values.yaml b/core/appmanager/chart/values.yaml
similarity index 100%
rename from appmanager/chart/values.yaml
rename to core/appmanager/chart/values.yaml
diff --git a/appmanager/cmd/main.go b/core/appmanager/cmd/main.go
similarity index 98%
rename from appmanager/cmd/main.go
rename to core/appmanager/cmd/main.go
index d4205d2..8d6695d 100644
--- a/appmanager/cmd/main.go
+++ b/core/appmanager/cmd/main.go
@@ -22,7 +22,7 @@
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
- app "github.com/giolekva/pcloud/appmanager"
+ app "github.com/giolekva/pcloud/core/appmanager"
)
var kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file.")
diff --git a/appmanager/go.mod b/core/appmanager/go.mod
similarity index 78%
rename from appmanager/go.mod
rename to core/appmanager/go.mod
index 284bf6d..96a749b 100644
--- a/appmanager/go.mod
+++ b/core/appmanager/go.mod
@@ -1,4 +1,4 @@
-module github.com/giolekva/pcloud/appmanager
+module github.com/giolekva/pcloud/core/appmanager
go 1.14
diff --git a/appmanager/go.sum b/core/appmanager/go.sum
similarity index 100%
rename from appmanager/go.sum
rename to core/appmanager/go.sum
diff --git a/appmanager/helm.go b/core/appmanager/helm.go
similarity index 100%
rename from appmanager/helm.go
rename to core/appmanager/helm.go
diff --git a/appmanager/installer.go b/core/appmanager/installer.go
similarity index 100%
rename from appmanager/installer.go
rename to core/appmanager/installer.go
diff --git a/appmanager/launcher.go b/core/appmanager/launcher.go
similarity index 100%
rename from appmanager/launcher.go
rename to core/appmanager/launcher.go
diff --git a/appmanager/manager.go b/core/appmanager/manager.go
similarity index 100%
rename from appmanager/manager.go
rename to core/appmanager/manager.go
diff --git a/appmanager/schema.go b/core/appmanager/schema.go
similarity index 100%
rename from appmanager/schema.go
rename to core/appmanager/schema.go
diff --git a/appmanager/triggers.go b/core/appmanager/triggers.go
similarity index 100%
rename from appmanager/triggers.go
rename to core/appmanager/triggers.go
diff --git a/appmanager/triggers_test.go b/core/appmanager/triggers_test.go
similarity index 100%
rename from appmanager/triggers_test.go
rename to core/appmanager/triggers_test.go
diff --git a/core/events/BUILD b/core/events/BUILD
new file mode 100644
index 0000000..78a9aee
--- /dev/null
+++ b/core/events/BUILD
@@ -0,0 +1,41 @@
+load("//:bazel_tools/docker.bzl", "docker_image")
+load("//:bazel_tools/helm.bzl", "helm_install", "helm_uninstall")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+
+# TODO(lekva): figure out how to build py_binary with pip dependencies and
+# migrate off docker_image rule
+docker_image(
+ name = "push_to_dev",
+ srcs = glob(
+ ["**"],
+ exclude = ["Dockerfile"],
+ ),
+ dockerfile = "Dockerfile",
+ image = "giolekva/pcloud-event-processor",
+ registry = "localhost:30500",
+ tag = "latest",
+)
+
+pkg_tar(
+ name = "chart",
+ srcs = glob(["chart/**"]),
+ extension = "tar.gz",
+ strip_prefix = "./chart",
+)
+
+helm_install(
+ name = "install",
+ args = {
+ "image.name": "localhost:30500/giolekva/pcloud-event-processor",
+ "image.pullPolicy": "Always",
+ },
+ chart = ":chart",
+ namespace = "pcloud-event-processor",
+ release_name = "init",
+)
+
+helm_uninstall(
+ name = "uninstall",
+ namespace = "pcloud-event-processor",
+ release_name = "init",
+)
diff --git a/events/Dockerfile b/core/events/Dockerfile
similarity index 74%
rename from events/Dockerfile
rename to core/events/Dockerfile
index 567c77c..1b8c159 100644
--- a/events/Dockerfile
+++ b/core/events/Dockerfile
@@ -7,9 +7,9 @@
ENV CGO_ENABLED 0
ENV GO111MODULE on
-RUN mkdir -p $GOPATH/src/github.com/giolekva/pcloud/events
-COPY . $GOPATH/src/github.com/giolekva/pcloud/events
-WORKDIR $GOPATH/src/github.com/giolekva/pcloud/events/cmd
+RUN mkdir -p $GOPATH/src/github.com/giolekva/pcloud/core/events
+COPY . $GOPATH/src/github.com/giolekva/pcloud/core/events
+WORKDIR $GOPATH/src/github.com/giolekva/pcloud/core/events/cmd
RUN go get ./...
RUN mkdir -p /app/build
diff --git a/events/appmanager.go b/core/events/appmanager.go
similarity index 100%
rename from events/appmanager.go
rename to core/events/appmanager.go
diff --git a/events/chart/Chart.yaml b/core/events/chart/Chart.yaml
similarity index 100%
rename from events/chart/Chart.yaml
rename to core/events/chart/Chart.yaml
diff --git a/events/chart/README.md b/core/events/chart/README.md
similarity index 100%
rename from events/chart/README.md
rename to core/events/chart/README.md
diff --git a/events/chart/templates/deployment.yaml b/core/events/chart/templates/deployment.yaml
similarity index 100%
rename from events/chart/templates/deployment.yaml
rename to core/events/chart/templates/deployment.yaml
diff --git a/events/chart/values.yaml b/core/events/chart/values.yaml
similarity index 100%
rename from events/chart/values.yaml
rename to core/events/chart/values.yaml
diff --git a/events/client.go b/core/events/client.go
similarity index 100%
rename from events/client.go
rename to core/events/client.go
diff --git a/events/cmd/main.go b/core/events/cmd/main.go
similarity index 96%
rename from events/cmd/main.go
rename to core/events/cmd/main.go
index c272a4e..a3944e7 100644
--- a/events/cmd/main.go
+++ b/core/events/cmd/main.go
@@ -7,7 +7,7 @@
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
- "github.com/giolekva/pcloud/events"
+ "github.com/giolekva/pcloud/core/events"
"github.com/golang/glog"
)
diff --git a/events/event.go b/core/events/event.go
similarity index 100%
rename from events/event.go
rename to core/events/event.go
diff --git a/events/go.mod b/core/events/go.mod
similarity index 90%
rename from events/go.mod
rename to core/events/go.mod
index 2cea9b0..e832718 100644
--- a/events/go.mod
+++ b/core/events/go.mod
@@ -1,4 +1,4 @@
-module github.com/giolekva/pcloud/events
+module github.com/giolekva/pcloud/core/events
go 1.14
diff --git a/events/go.sum b/core/events/go.sum
similarity index 100%
rename from events/go.sum
rename to core/events/go.sum
diff --git a/events/processor.go b/core/events/processor.go
similarity index 100%
rename from events/processor.go
rename to core/events/processor.go
diff --git a/dev/create_dev_cluster.sh b/dev/create_dev_cluster.sh
index e6b01e1..54e66af 100755
--- a/dev/create_dev_cluster.sh
+++ b/dev/create_dev_cluster.sh
@@ -1,6 +1,7 @@
#!/bin/bash
-ROOT="$(dirname -- $(pwd))"
+ROOT=$(pwd)
+ROOT=${ROOT%/pcloud*}/pcloud
k3d cluster create pcloud-dev \
--servers=1 \
@@ -12,8 +13,7 @@
# Traefik
helm repo add traefik https://containous.github.io/traefik-helm-chart
helm repo update
-kubectl create namespace traefik
-helm --namespace=traefik install traefik traefik/traefik \
+helm --namespace=traefik install --create-namespace traefik traefik/traefik \
--set additionalArguments="{--providers.kubernetesingress,--global.checknewversion=true}" \
--set ports.traefik.expose=True
diff --git a/dev/install_core_services.sh b/dev/install_core_services.sh
index 339a014..1ab542a 100755
--- a/dev/install_core_services.sh
+++ b/dev/install_core_services.sh
@@ -1,19 +1,20 @@
#!/bin/sh
-ROOT="$(dirname -- $(pwd))"
+ROOT=$(pwd)
+ROOT=${ROOT%/pcloud*}/pcloud
# Dgraph
source $ROOT/apps/dgraph/install.sh
# Application Manager
-bazel run //appmanager:push_to_dev
-bazel run //appmanager:install
+bazel run //core/appmanager:push_to_dev
+bazel run //core/appmanager:install
# Event Processor
-bazel run //events:push_to_dev
-bazel run //events:install
+bazel run //core/events:push_to_dev
+bazel run //core/events:install
# Knowledge Graph
-bazel run //controller:push_to_dev
+bazel run //core/api:push_to_dev
source $ROOT/dev/bootstrap_schema.sh
-bazel run //controller:install
+bazel run //core/api:install
diff --git a/docs/dev.md b/docs/dev.md
index 7f6553f..b858ebe 100644
--- a/docs/dev.md
+++ b/docs/dev.md
@@ -2,14 +2,40 @@
## Prerequisites
PCloud uses following tools to build and deploy it's packages:
-* Docker - To build container images: https://helm.sh/docs/intro/install/
+* Docker - To build container images: https://docs.docker.com/get-docker/
* k3d - To create local Kubernetes cluster for development environment: https://k3d.io/#installation
* kubectl - To interacto with the cluster: https://kubectl.docs.kubernetes.io/installation/kubectl/
* Helm - To package and distributes both PCloud core services and applications running on it: https://helm.sh/docs/intro/install/
Each of these tools provide multiple ways of installing them, choose the one which best suits you and your Host OS.
+# Development Instructions
+
## Installing PCloud
PCloud installation is two step process:
-1) First local Kubernetes cluster has to be created: ./dev/create_dev_cluster.sh
-2) Now we can install PCloud core services top of it: ./dev/install_core_services.sh
+```shell
+> ./dev/create_dev_cluster.sh # creates local Kubernetes cluster
+> ./dev/install_core_services.sh # installs PCloud core services
+```
+
+## Installing applications
+Under apps/ directory one can find number of sample application for PCloud. Installing any of those requires building container image, creating Helm Chart tarball and uploading it to Application Manager.
+Let's see what that looks like for rpuppy application:
+```shell
+> bazel run //apps/rpuppy:push_to_dev # builds and pushes container image to localhost:30500 container registry, which is running inside PCloud cluster
+> bazel build //apps/rpuppy:chart # creates Helm Chart tartball which can be found at bazel-bin/apps/rpuppy/chart.tar.gz
+```
+
+## Redeploying core services
+To redeploy one of the core services after making changes in it you have to rebuild container images and restart the running service. Let's see what that looks like for API Service:
+```shell
+> bazel run //core/api:push_to_dev # builds and pushes container image to localhost:30500 container registry, which is running inside PCloud cluster
+> kubectl -n pcloud rollout restart deployment/api
+```
+
+Reflecting changes in the chart configuration requires reinstallation of the chart itself:
+```shell
+> bazel build //core/api:chart # creates chart tarball
+> bazel run //core/api:uninstall # uninstalls old Helm Chart from the cluster
+> bazel run //core/api:install # installs new Helm Chart to the cluster
+```
diff --git a/events/BUILD b/events/BUILD
deleted file mode 100644
index 71bacea..0000000
--- a/events/BUILD
+++ /dev/null
@@ -1,32 +0,0 @@
-load("//:bazel_tools/docker.bzl", "docker_image")
-load("//:bazel_tools/helm.bzl", "helm_install")
-load("@rules_pkg//:pkg.bzl", "pkg_tar")
-
-# TODO(lekva): figure out how to build py_binary with pip dependencies and
-# migrate off docker_image rule
-docker_image(
- name = "push_to_dev",
- registry = "localhost:30500",
- image = "giolekva/pcloud-event-processor",
- tag = "latest",
- dockerfile = "Dockerfile",
- srcs = glob(["**"], exclude=["Dockerfile"]),
-)
-
-pkg_tar(
- name = "chart",
- srcs = glob(["chart/**"]),
- extension = "tar.gz",
- strip_prefix = "./chart",
-)
-
-helm_install(
- name = "install",
- namespace = "pcloud-event-processor",
- release_name = "init",
- chart = ":chart",
- args = {
- "image.name": "localhost:30500/giolekva/pcloud-event-processor",
- "image.pullPolicy": "Always",
- },
-)