Canvas: Generate graph state out of dodo-app config
Restructure code, create shared config lib.
Change-Id: I2cf06d35c486d4557484daf8618a2c215316fa7e
diff --git a/apps/canvas/back/.env b/apps/canvas/back/.env
index 6237047..a101621 100644
--- a/apps/canvas/back/.env
+++ b/apps/canvas/back/.env
@@ -1,3 +1,3 @@
-DATABASE_URL=file:${DODO_VOLUME_DATA}/dodo.db
-PUBLIC_ADDR=https://canvas.v1.dodo.cloud
-INTERNAL_API_ADDR=http://canvas-app.hgrz-dodo-app-gry.svc.cluster.local:8081
+DATABASE_URL=file:/home/gio/dodo.db
+PUBLIC_ADDR=https://canvas.p.v1.dodo.cloud
+INTERNAL_API_ADDR=http://canvas.hgrz-dodo-app-jjy.svc.cluster.local:8081
diff --git a/apps/canvas/back/package-lock.json b/apps/canvas/back/package-lock.json
index cd69cf4..03f0e80 100644
--- a/apps/canvas/back/package-lock.json
+++ b/apps/canvas/back/package-lock.json
@@ -12,6 +12,7 @@
"@loancrate/prisma-schema-parser": "^3.0.0",
"@prisma/client": "^6.6.0",
"axios": "^1.8.4",
+ "config": "file:../config",
"dotenv": "^16.5.0",
"dotenv-expand": "^12.0.2",
"express": "^4.21.1",
@@ -40,6 +41,21 @@
"typescript-eslint": "^8.11.0"
}
},
+ "../config": {
+ "version": "1.0.0",
+ "license": "ISC",
+ "dependencies": {
+ "@xyflow/react": "^12.3.3",
+ "uuid": "^11.0.2",
+ "zod": "^3.24.4"
+ },
+ "devDependencies": {
+ "eslint": "^9.13.0",
+ "prettier": "3.5.3",
+ "typescript": "^5.8.3",
+ "typescript-eslint": "^8.11.0"
+ }
+ },
"node_modules/@ampproject/remapping": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz",
@@ -4279,6 +4295,10 @@
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"devOptional": true
},
+ "node_modules/config": {
+ "resolved": "../config",
+ "link": true
+ },
"node_modules/console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
diff --git a/apps/canvas/back/package.json b/apps/canvas/back/package.json
index 5f69e57..b32b801 100644
--- a/apps/canvas/back/package.json
+++ b/apps/canvas/back/package.json
@@ -3,7 +3,7 @@
"version": "1.0.0",
"description": "",
"main": "index.js",
- "type": "commonjs",
+ "type": "module",
"scripts": {
"build": "tsc",
"test": "jest",
@@ -18,6 +18,7 @@
"@loancrate/prisma-schema-parser": "^3.0.0",
"@prisma/client": "^6.6.0",
"axios": "^1.8.4",
+ "config": "file:../config",
"dotenv": "^16.5.0",
"dotenv-expand": "^12.0.2",
"express": "^4.21.1",
diff --git a/apps/canvas/back/src/app_manager.ts b/apps/canvas/back/src/app_manager.ts
index 803192c..faf64e2 100644
--- a/apps/canvas/back/src/app_manager.ts
+++ b/apps/canvas/back/src/app_manager.ts
@@ -69,7 +69,6 @@
if (response.status !== 200) {
throw new Error(`Failed to deploy application: ${response.statusText}`);
}
- console.log(response.data);
const result = DeployResponseSchema.safeParse(response.data);
if (!result.success) {
throw new Error(`Invalid deploy response format: ${result.error.message}`);
diff --git a/apps/canvas/back/src/index.ts b/apps/canvas/back/src/index.ts
index a2e8b31..0ba25b0 100644
--- a/apps/canvas/back/src/index.ts
+++ b/apps/canvas/back/src/index.ts
@@ -3,15 +3,16 @@
import fs from "node:fs";
import { env } from "node:process";
import axios from "axios";
-import { GithubClient } from "./github";
-import { AppManager } from "./app_manager";
+import { GithubClient } from "./github.js";
+import { AppManager } from "./app_manager.js";
import { z } from "zod";
-import { ProjectMonitor, WorkerSchema } from "./project_monitor";
+import { ProjectMonitor, WorkerSchema } from "./project_monitor.js";
import tmp from "tmp";
-import { NodeJSAnalyzer } from "./lib/nodejs";
+import { NodeJSAnalyzer } from "./lib/nodejs.js";
import shell from "shelljs";
-import { RealFileSystem } from "./lib/fs";
+import { RealFileSystem } from "./lib/fs.js";
import path from "node:path";
+import { Env, generateDodoConfig, ConfigSchema, AppNode, ConfigWithInput, configToGraph, Network } from "config";
async function generateKey(root: string): Promise<[string, string]> {
const privKeyPath = path.join(root, "key");
@@ -120,31 +121,44 @@
}
resp.status(200);
resp.header("content-type", "application/json");
+ let currentState: Record<string, unknown> | null = null;
if (state === "deploy") {
if (r.state == null) {
- resp.send({
+ currentState = {
nodes: [],
edges: [],
viewport: { x: 0, y: 0, zoom: 1 },
- });
+ };
} else {
- resp.send(JSON.parse(Buffer.from(r.state).toString("utf8")));
+ currentState = JSON.parse(Buffer.from(r.state).toString("utf8"));
}
} else {
if (r.draft == null) {
if (r.state == null) {
- resp.send({
+ currentState = {
nodes: [],
edges: [],
viewport: { x: 0, y: 0, zoom: 1 },
- });
+ };
} else {
- resp.send(JSON.parse(Buffer.from(r.state).toString("utf8")));
+ currentState = JSON.parse(Buffer.from(r.state).toString("utf8"));
}
} else {
- resp.send(JSON.parse(Buffer.from(r.draft).toString("utf8")));
+ currentState = JSON.parse(Buffer.from(r.draft).toString("utf8"));
}
}
+ const env = await getEnv(Number(req.params["projectId"]), resp.locals.userId, resp.locals.username);
+ if (currentState) {
+ const config = generateDodoConfig(
+ req.params["projectId"].toString(),
+ currentState.nodes as AppNode[],
+ env,
+ );
+ resp.send({
+ state: currentState,
+ config,
+ });
+ }
} catch (e) {
console.log(e);
resp.status(500);
@@ -257,7 +271,6 @@
deployKey: string,
publicAddr?: string,
): Promise<void> {
- console.log(publicAddr);
for (const repoUrl of diff.toDelete ?? []) {
try {
await github.removeDeployKey(repoUrl, deployKey);
@@ -289,11 +302,10 @@
const handleDeploy: express.Handler = async (req, resp) => {
try {
const projectId = Number(req.params["projectId"]);
- const state = JSON.stringify(req.body.state);
const p = await db.project.findUnique({
where: {
id: projectId,
- userId: resp.locals.userId,
+ // userId: resp.locals.userId, TODO(gio): validate
},
select: {
instanceId: true,
@@ -307,6 +319,21 @@
resp.status(404);
return;
}
+ const config = ConfigSchema.safeParse(req.body.config);
+ if (!config.success) {
+ resp.status(400);
+ resp.write(JSON.stringify({ error: "Invalid configuration", issues: config.error.format() }));
+ return;
+ }
+ const state = req.body.state
+ ? JSON.stringify(req.body.state)
+ : JSON.stringify(
+ configToGraph(
+ config.data,
+ getNetworks(resp.locals.username),
+ p.state ? JSON.parse(Buffer.from(p.state).toString("utf8")) : null,
+ ),
+ );
await db.project.update({
where: {
id: projectId,
@@ -325,14 +352,20 @@
});
}
let diff: RepoDiff | null = null;
- const config = req.body.config;
- config.input.key = {
- public: deployKeyPublic,
- private: deployKey,
+ const cfg: ConfigWithInput = {
+ ...config.data,
+ input: {
+ appId: projectId.toString(),
+ managerAddr: env.INTERNAL_API_ADDR!,
+ key: {
+ public: deployKeyPublic!,
+ private: deployKey!,
+ },
+ },
};
try {
if (p.instanceId == null) {
- const deployResponse = await appManager.deploy(config);
+ const deployResponse = await appManager.deploy(cfg);
await db.project.update({
where: {
id: projectId,
@@ -346,7 +379,7 @@
});
diff = { toAdd: extractGithubRepos(state) };
} else {
- const deployResponse = await appManager.update(p.instanceId, config);
+ const deployResponse = await appManager.update(p.instanceId, cfg);
diff = calculateRepoDiff(extractGithubRepos(p.state), extractGithubRepos(state));
await db.project.update({
where: {
@@ -413,6 +446,41 @@
}
};
+const handleConfigGet: express.Handler = async (req, resp) => {
+ try {
+ const projectId = Number(req.params["projectId"]);
+ const project = await db.project.findUnique({
+ where: {
+ id: projectId,
+ },
+ select: {
+ state: true,
+ },
+ });
+
+ if (!project || !project.state) {
+ resp.status(404).send({ error: "No deployed configuration found." });
+ return;
+ }
+
+ const state = JSON.parse(Buffer.from(project.state).toString("utf8"));
+ const env = await getEnv(projectId, resp.locals.userId, resp.locals.username);
+ const config = generateDodoConfig(projectId.toString(), state.nodes, env);
+
+ if (!config) {
+ resp.status(500).send({ error: "Failed to generate configuration." });
+ return;
+ }
+ resp.status(200).json(config);
+ } catch (e) {
+ console.log(e);
+ resp.status(500).send({ error: "Internal server error" });
+ } finally {
+ console.log("config get done");
+ resp.end();
+ }
+};
+
const handleRemoveDeployment: express.Handler = async (req, resp) => {
try {
const projectId = Number(req.params["projectId"]);
@@ -529,80 +597,84 @@
}
};
+const getNetworks = (username?: string | undefined): Network[] => {
+ return [
+ {
+ name: "Trial",
+ domain: "trial.dodoapp.xyz",
+ hasAuth: false,
+ },
+ // TODO(gio): Remove
+ ].concat(
+ username === "gio" || 1 == 1
+ ? [
+ {
+ name: "Public",
+ domain: "v1.dodo.cloud",
+ hasAuth: true,
+ },
+ {
+ name: "Private",
+ domain: "p.v1.dodo.cloud",
+ hasAuth: true,
+ },
+ ]
+ : [],
+ );
+};
+
+const getEnv = async (projectId: number, userId: string, username: string): Promise<Env> => {
+ const project = await db.project.findUnique({
+ where: {
+ id: projectId,
+ userId,
+ },
+ select: {
+ deployKeyPublic: true,
+ githubToken: true,
+ access: true,
+ instanceId: true,
+ },
+ });
+ if (!project) {
+ throw new Error("Project not found");
+ }
+ const monitor = projectMonitors.get(projectId);
+ const serviceNames = monitor ? monitor.getAllServiceNames() : [];
+ const services = serviceNames.map((name: string) => ({
+ name,
+ workers: [...(monitor ? monitor.getWorkerStatusesForService(name) : new Map()).entries()].map(
+ ([id, status]) => ({
+ ...status,
+ id,
+ }),
+ ),
+ }));
+ return {
+ managerAddr: env.INTERNAL_API_ADDR,
+ deployKeyPublic: project.deployKeyPublic == null ? undefined : project.deployKeyPublic,
+ instanceId: project.instanceId == null ? undefined : project.instanceId,
+ access: JSON.parse(project.access ?? "[]"),
+ integrations: {
+ github: !!project.githubToken,
+ },
+ networks: getNetworks(username),
+ services,
+ user: {
+ id: userId,
+ username: username,
+ },
+ };
+};
+
const handleEnv: express.Handler = async (req, resp) => {
const projectId = Number(req.params["projectId"]);
try {
- const project = await db.project.findUnique({
- where: {
- id: projectId,
- userId: resp.locals.userId,
- },
- select: {
- deployKeyPublic: true,
- githubToken: true,
- access: true,
- instanceId: true,
- },
- });
- if (!project) {
- resp.status(404);
- resp.write(JSON.stringify({ error: "Project not found" }));
- return;
- }
- const monitor = projectMonitors.get(projectId);
- const serviceNames = monitor ? monitor.getAllServiceNames() : [];
- const services = serviceNames.map((name) => ({
- name,
- workers: [...(monitor ? monitor.getWorkerStatusesForService(name) : new Map()).entries()].map(
- ([id, status]) => ({
- ...status,
- id,
- }),
- ),
- }));
-
+ const env = await getEnv(projectId, resp.locals.userId, resp.locals.username);
resp.status(200);
- resp.write(
- JSON.stringify({
- managerAddr: env.INTERNAL_API_ADDR,
- deployKeyPublic: project.deployKeyPublic == null ? undefined : project.deployKeyPublic,
- instanceId: project.instanceId == null ? undefined : project.instanceId,
- access: JSON.parse(project.access ?? "[]"),
- integrations: {
- github: !!project.githubToken,
- },
- networks: [
- {
- name: "Trial",
- domain: "trial.dodoapp.xyz",
- hasAuth: false,
- },
- // TODO(gio): Remove
- ].concat(
- resp.locals.username !== "gio"
- ? []
- : [
- {
- name: "Public",
- domain: "v1.dodo.cloud",
- hasAuth: true,
- },
- {
- name: "Private",
- domain: "p.v1.dodo.cloud",
- hasAuth: true,
- },
- ],
- ),
- services,
- user: {
- id: resp.locals.userId,
- username: resp.locals.username,
- },
- }),
- );
+ resp.write(JSON.stringify(env));
} catch (error) {
- console.error("Error checking integrations:", error);
+ console.error("Error getting env:", error);
resp.status(500);
resp.write(JSON.stringify({ error: "Internal server error" }));
} finally {
@@ -692,12 +764,17 @@
return true;
}
const results = await Promise.all(
- projectWorkers.map(async (workerAddress) => {
- const resp = await axios.post(`${workerAddress}/update`);
- return resp.status === 200;
+ projectWorkers.map(async (workerAddress: string) => {
+ try {
+ const { data } = await axios.get(`http://${workerAddress}/reload`);
+ return data.every((s: { status: string }) => s.status === "ok");
+ } catch (error) {
+ console.error(`Failed to reload worker ${workerAddress}:`, error);
+ return false;
+ }
}),
);
- return results.reduce((acc, curr) => acc && curr, true);
+ return results.reduce((acc: boolean, curr: boolean) => acc && curr, true);
}
const handleReload: express.Handler = async (req, resp) => {
@@ -886,9 +963,30 @@
}
};
+const handleValidateConfig: express.Handler = async (req, resp) => {
+ try {
+ const validationResult = ConfigSchema.safeParse(req.body);
+ if (!validationResult.success) {
+ resp.status(400);
+ resp.header("Content-Type", "application/json");
+ resp.write(JSON.stringify({ success: false, errors: validationResult.error.flatten() }));
+ } else {
+ resp.status(200);
+ resp.header("Content-Type", "application/json");
+ resp.write(JSON.stringify({ success: true }));
+ }
+ } catch (e) {
+ console.log(e);
+ resp.status(500);
+ } finally {
+ resp.end();
+ }
+};
+
async function start() {
await db.$connect();
const app = express();
+ app.set("json spaces", 2);
app.use(express.json()); // Global JSON parsing
// Public webhook route - no auth needed
@@ -903,6 +1001,7 @@
projectRouter.get("/:projectId/saved/draft", handleSavedGet("draft"));
projectRouter.post("/:projectId/deploy", handleDeploy);
projectRouter.get("/:projectId/status", handleStatus);
+ projectRouter.get("/:projectId/config", handleConfigGet);
projectRouter.delete("/:projectId", handleProjectDelete);
projectRouter.get("/:projectId/repos/github", handleGithubRepos);
projectRouter.post("/:projectId/github-token", handleUpdateGithubToken);
@@ -921,6 +1020,9 @@
const internalApi = express();
internalApi.use(express.json());
internalApi.post("/api/project/:projectId/workers", handleRegisterWorker);
+ internalApi.get("/api/project/:projectId/config", handleConfigGet);
+ internalApi.post("/api/project/:projectId/deploy", handleDeploy);
+ internalApi.post("/api/validate-config", handleValidateConfig);
app.listen(env.DODO_PORT_WEB, () => {
console.log("Web server started on port", env.DODO_PORT_WEB);
diff --git a/apps/canvas/back/src/lib/analyze.ts b/apps/canvas/back/src/lib/analyze.ts
index e1fa699..28eecbf 100644
--- a/apps/canvas/back/src/lib/analyze.ts
+++ b/apps/canvas/back/src/lib/analyze.ts
@@ -1,4 +1,4 @@
-import { FileSystem } from "./fs";
+import { FileSystem } from "./fs.js";
export interface ServiceAnalyzer {
detect: (fs: FileSystem, root: string) => boolean;
diff --git a/apps/canvas/back/src/lib/nodejs.test.ts b/apps/canvas/back/src/lib/nodejs.test.ts
index 7d406b1..5de8423 100644
--- a/apps/canvas/back/src/lib/nodejs.test.ts
+++ b/apps/canvas/back/src/lib/nodejs.test.ts
@@ -1,8 +1,8 @@
-import { NodeJSAnalyzer } from "./nodejs";
-import { FileSystem, RealFileSystem } from "./fs";
+import { NodeJSAnalyzer } from "./nodejs.js";
+import { FileSystem, RealFileSystem } from "./fs.js";
import { Volume, IFs, createFsFromVolume } from "memfs";
import { test, expect } from "@jest/globals";
-import { expandValue } from "./env";
+import { expandValue } from "./env.js";
import shell from "shelljs";
class InMemoryFileSystem implements FileSystem {
diff --git a/apps/canvas/back/src/lib/nodejs.ts b/apps/canvas/back/src/lib/nodejs.ts
index 07e6c1f..3369b1d 100644
--- a/apps/canvas/back/src/lib/nodejs.ts
+++ b/apps/canvas/back/src/lib/nodejs.ts
@@ -1,10 +1,10 @@
import path from "path";
-import { FileSystem } from "./fs";
-import { ServiceAnalyzer, ConfigVar, ConfigVarCategory, ConfigVarSemanticType } from "./analyze";
+import { FileSystem } from "./fs.js";
+import { ServiceAnalyzer, ConfigVar, ConfigVarCategory, ConfigVarSemanticType } from "./analyze.js";
import { parse as parseDotenv } from "dotenv";
import { parsePrismaSchema } from "@loancrate/prisma-schema-parser";
-import { augmentConfigVar } from "./semantics";
-import { expandValue } from "./env";
+import { augmentConfigVar } from "./semantics.js";
+import { expandValue } from "./env.js";
import { z } from "zod";
const packageJsonFileName = "package.json";
diff --git a/apps/canvas/back/src/lib/semantics.ts b/apps/canvas/back/src/lib/semantics.ts
index 2cbd1ab..f365f83 100644
--- a/apps/canvas/back/src/lib/semantics.ts
+++ b/apps/canvas/back/src/lib/semantics.ts
@@ -1,4 +1,4 @@
-import { ConfigVar, ConfigVarSemanticType } from "./analyze";
+import { ConfigVar, ConfigVarSemanticType } from "./analyze.js";
export function augmentConfigVar(cv: ConfigVar) {
if (cv.semanticType != null) {
diff --git a/apps/canvas/back/tsconfig.json b/apps/canvas/back/tsconfig.json
index c0ce9b7..b92e830 100644
--- a/apps/canvas/back/tsconfig.json
+++ b/apps/canvas/back/tsconfig.json
@@ -13,7 +13,7 @@
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
- "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+ "target": "es2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
@@ -26,7 +26,10 @@
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
- "module": "commonjs" /* Specify what module code is generated. */,
+ "module": "node16",
+ "moduleResolution": "node16",
+ "allowImportingTsExtensions": false,
+ "noEmit": false,
// "rootDir": "./", /* Specify the root folder within your source files. */
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
@@ -49,7 +52,7 @@
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
- // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+ "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */