diff --git a/README.md b/README.md index f780d753e..3b9d4a37f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Open the Command Palette (`Cmd+Shift+P` or `Ctrl+Shift+P`) and type `Deepnote` t | `Deepnote: Set API Token` | Save a Deepnote API token for syncing and publishing | | `Deepnote: Clear API Token` | Remove the saved Deepnote API token | | `Deepnote: Sync with Deepnote Cloud` | Sync a local folder with your Deepnote workspace | +| `Deepnote: Publish App…` | Publish a folder, such as an app's build output, as a Deepnote app | | `Notebook: Select Notebook Kernel` | Select or switch kernels within your notebook | | `Notebook: Change Cell Language` | Change the language of the cell currently in focus | | `Deepnote: Enable Snapshots` | Enable snapshot mode for the current workspace | @@ -94,6 +95,20 @@ Known limitations: - After a sync updates an open notebook, the editor can save the notebook again, and the next sync can then push that project or report a conflict for it. - A folder reached through a symbolic link or junction is checked by its path in the workspace, not by its target, so a link into another synced folder is not refused. +### Publishing apps + +Run `Deepnote: Publish App…` from the Command Palette, or right-click a folder in the file explorer and choose `Publish App…`. It uploads the folder to a Deepnote project as an app, then shows the app URL with "Open" and "Copy". + +- The folder is published as it is. Build your app first, then publish its build output folder, such as `dist/`. +- Each file of the folder replaces the app file at the same path in Deepnote. Files you delete from the folder are not deleted in Deepnote and stay published; delete them in Deepnote. +- When the folder is inside a synced project, the app goes to that project. Otherwise you choose a project of your Deepnote workspace. +- You also choose whether the app can call the Deepnote API. With API access, the app calls Deepnote with a short-lived token scoped to the viewer, never your personal token. +- The choices are remembered per folder, so publishing the folder again asks only for one confirmation, with "Change Settings…" to choose again. +- Publishing uses `DEEPNOTE_TOKEN` from a `.env` file in a synced folder above the app folder, or else the token saved with `Deepnote: Set API Token`. A folder that contains a `.env` or `.env.*` file cannot be published, because every published file is public. +- Publishing into a synced folder does not start while a sync or another such publish is in progress. +- For a folder inside a synced project, when app files changed in Deepnote since the last sync or publish, publishing asks first. "Force" overwrites them in Deepnote. "Sync First" syncs the whole synced folder with the working files of every project, which downloads Deepnote's versions of the app files into the synced project's `.files/_deepnote_static` folder, so you can merge them into your app and publish again. A project with a working file over 100 MiB fails to sync this way. A folder outside a synced project overwrites app files changed in Deepnote without asking. +- Publishing needs a trusted workspace and the desktop app. + ### Database integrations Configure database connections for SQL blocks: diff --git a/package.json b/package.json index 2d723254e..5c9e6a207 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,13 @@ "icon": "$(sync)", "enablement": "isWorkspaceTrusted && !deepnote.webExtension" }, + { + "command": "deepnote.publishApp", + "title": "%deepnote.commands.publishApp.title%", + "category": "Deepnote", + "icon": "$(rocket)", + "enablement": "isWorkspaceTrusted && !deepnote.webExtension" + }, { "command": "deepnote.newProject", "title": "%deepnote.commands.newProject.title%", @@ -1249,6 +1256,11 @@ "when": "resourceExtname == .ipynb && isWorkspaceTrusted", "command": "deepnote.importnotebookfile", "group": "Deepnote" + }, + { + "when": "explorerResourceIsFolder && isWorkspaceTrusted && !deepnote.webExtension", + "command": "deepnote.publishApp", + "group": "Deepnote" } ], "commandPalette": [ diff --git a/package.nls.json b/package.nls.json index 1312923d9..b7c5f4661 100644 --- a/package.nls.json +++ b/package.nls.json @@ -262,6 +262,7 @@ "deepnote.commands.setApiToken.title": "Set API Token", "deepnote.commands.clearApiToken.title": "Clear API Token", "deepnote.commands.syncWithDeepnoteCloud.title": "Sync with Deepnote Cloud", + "deepnote.commands.publishApp.title": "Publish App…", "deepnote.commands.newProject.title": "New Project", "deepnote.commands.importNotebook.title": "Import Notebook", "deepnote.commands.importJupyterNotebook.title": "Import Jupyter Notebook", diff --git a/src/commands.ts b/src/commands.ts index fdc33c76c..892fb2092 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -16,6 +16,7 @@ import { PythonEnvironment } from './platform/pythonEnvironments/info'; import { Channel } from './platform/common/application/types'; import { IJupyterVariable } from './kernels/variables/types'; import { IShowDataViewerFromVariablePanel } from './messageTypes'; +import { SyncWithDeepnoteCloudTarget } from './notebooks/deepnote/cloud/types'; export type CommandIds = keyof ICommandNameArgumentTypeMapping; @@ -209,5 +210,6 @@ export interface ICommandNameArgumentTypeMapping { [DSCommands.OpenInDeepnote]: []; [DSCommands.SetApiToken]: [Uri | undefined]; [DSCommands.ClearApiToken]: []; - [DSCommands.SyncWithDeepnoteCloud]: []; + [DSCommands.SyncWithDeepnoteCloud]: [SyncWithDeepnoteCloudTarget | undefined]; + [DSCommands.PublishApp]: [Uri | undefined]; } diff --git a/src/notebooks/deepnote/cloud/deepnoteCloudOperationLock.ts b/src/notebooks/deepnote/cloud/deepnoteCloudOperationLock.ts new file mode 100644 index 000000000..b9885ba4f --- /dev/null +++ b/src/notebooks/deepnote/cloud/deepnoteCloudOperationLock.ts @@ -0,0 +1,21 @@ +import { injectable } from 'inversify'; +import { Disposable } from 'vscode'; + +import { IDeepnoteCloudOperationLock } from './types'; + +@injectable() +export class DeepnoteCloudOperationLock implements IDeepnoteCloudOperationLock { + private held = false; + + public tryAcquire(): Disposable | undefined { + if (this.held) { + return undefined; + } + + this.held = true; + + return new Disposable(() => { + this.held = false; + }); + } +} diff --git a/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.ts b/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.ts new file mode 100644 index 000000000..b69520bcb --- /dev/null +++ b/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.ts @@ -0,0 +1,402 @@ +import { listAllProjects, PROJECT_STATIC_ROOT, type SyncProject } from '@deepnote/cloud'; +import { + findSyncManifestRoot, + loadSyncManifest, + projectFilesDir, + publishApp, + PublishDivergedError, + type PublishAppEvent, + type PublishAppOptions, + type PublishAppResult, + type SyncManifest, + type WorkspaceSyncResult +} from '@deepnote/cloud-sync'; +import { inject, injectable } from 'inversify'; +import { commands, Disposable, Progress, ProgressLocation, QuickPickItem, Uri, window, workspace } from 'vscode'; + +import { IExtensionSyncActivationService } from '../../../platform/activation/types'; +import { Commands } from '../../../platform/common/constants'; +import { getFilePath } from '../../../platform/common/platform/fs-paths'; +import { IExtensionContext } from '../../../platform/common/types'; +import { DeepnoteCloud } from '../../../platform/common/utils/localize'; +import { logger } from '../../../platform/logging'; +import { basename, relativePath } from '../../../platform/vscode-path/resources'; +import { getApiEndpoint } from '../importClient.node'; +import { showCloudError } from './cloudErrors.node'; +import { showPublishedAppUrl } from './publishedAppUrl.node'; +import { IDeepnoteApiTokenService, IDeepnoteCloudOperationLock, SyncWithDeepnoteCloudTarget } from './types'; + +const SETTINGS_STATE_KEY = 'deepnote.publishApp.folderSettings'; +const MAX_LISTED_PATHS = 10; + +type PublishAppSettings = { projectId: string; projectName: string; apiAccess: boolean }; +type ProjectChoice = Pick; + +interface ProjectPickItem extends QuickPickItem { + projectId: string; +} + +interface ApiAccessPickItem extends QuickPickItem { + apiAccess: boolean; +} + +/** `showQuickPick` cannot preselect an item, so the current choice is listed first instead. */ +function currentFirst(items: T[], isCurrent: (item: T) => boolean): T[] { + return [...items.filter(isCurrent), ...items.filter((item) => !isCurrent(item))]; +} + +function listPaths(paths: string[]): string { + const listed = paths.slice(0, MAX_LISTED_PATHS).join(', '); + const rest = paths.length - MAX_LISTED_PATHS; + + return rest > 0 ? `${listed} ${DeepnoteCloud.andMore(rest)}` : listed; +} + +@injectable() +export class PublishAppCommandHandler implements IExtensionSyncActivationService { + constructor( + @inject(IDeepnoteApiTokenService) private readonly apiTokenService: IDeepnoteApiTokenService, + @inject(IDeepnoteCloudOperationLock) private readonly operationLock: IDeepnoteCloudOperationLock, + @inject(IExtensionContext) private readonly extensionContext: IExtensionContext + ) {} + + public activate(): void { + this.extensionContext.subscriptions.push( + commands.registerCommand(Commands.PublishApp, (arg?: unknown) => + this.publishFromCommand(arg instanceof Uri ? arg : undefined) + ) + ); + } + + /** Seam for tests, which never reach the network. */ + protected listProjects(baseUrl: string, token: string): Promise { + return listAllProjects(baseUrl, token); + } + + /** Seam for tests, which never reach the network. */ + protected runPublishApp(options: PublishAppOptions): Promise { + return publishApp(options); + } + + private async chooseApiAccess(current: boolean | undefined): Promise { + const items = ( + [ + { label: DeepnoteCloud.apiAccessOff, detail: DeepnoteCloud.apiAccessOffDetail, apiAccess: false }, + { label: DeepnoteCloud.apiAccessOn, detail: DeepnoteCloud.apiAccessOnDetail, apiAccess: true } + ] satisfies ApiAccessPickItem[] + ).map((item) => + item.apiAccess === current ? { ...item, detail: `${DeepnoteCloud.currentChoice}. ${item.detail}` } : item + ); + const picked = await window.showQuickPick( + currentFirst(items, (item) => item.apiAccess === current), + { placeHolder: DeepnoteCloud.chooseApiAccess } + ); + + return picked?.apiAccess; + } + + private async chooseFolder(): Promise { + const [folder] = + (await window.showOpenDialog({ + canSelectFolders: true, + canSelectFiles: false, + canSelectMany: false, + defaultUri: workspace.workspaceFolders?.[0]?.uri, + title: DeepnoteCloud.chooseAppFolderTitle, + openLabel: DeepnoteCloud.publish + })) ?? []; + + return folder; + } + + private async chooseProject( + folder: Uri, + token: string, + current: PublishAppSettings | undefined + ): Promise { + const synced = await this.findSyncedProject(folder); + if (synced) { + return synced; + } + + let projects: SyncProject[]; + try { + projects = await window.withProgress( + { location: ProgressLocation.Notification, title: DeepnoteCloud.loadingProjects }, + () => this.listProjects(getApiEndpoint(folder), token) + ); + } catch (error) { + void showCloudError(DeepnoteCloud.loadProjectsFailed, error, folder); + + return undefined; + } + + const isCurrent = (item: ProjectPickItem) => item.projectId === current?.projectId; + const items = [...projects] + .sort((a, b) => a.name.localeCompare(b.name)) + .map((project): ProjectPickItem => ({ + label: project.name, + description: project.folder?.path.map((segment) => segment.name).join(' / '), + projectId: project.id + })) + .map((item) => (isCurrent(item) ? { ...item, detail: DeepnoteCloud.currentChoice } : item)); + const picked = await window.showQuickPick(currentFirst(items, isCurrent), { + placeHolder: DeepnoteCloud.chooseProject + }); + + return picked && { projectId: picked.projectId, projectName: picked.label }; + } + + private async collectSettings( + folder: Uri, + token: string, + current?: PublishAppSettings + ): Promise { + const project = await this.chooseProject(folder, token, current); + if (!project) { + return undefined; + } + + const apiAccess = await this.chooseApiAccess(current?.apiAccess); + if (apiAccess === undefined) { + return undefined; + } + + const settings: PublishAppSettings = { ...project, apiAccess }; + await this.extensionContext.workspaceState.update(SETTINGS_STATE_KEY, { + ...this.savedSettings(), + [folder.toString()]: settings + }); + + return settings; + } + + private async findSyncedProject(folder: Uri): Promise { + let root: string | undefined; + let manifest: SyncManifest; + try { + root = await findSyncManifestRoot(getFilePath(folder)); + if (!root) { + return undefined; + } + manifest = await loadSyncManifest(root); + } catch (error) { + logger.warn('Could not read the sync manifest above the app folder', error); + + return undefined; + } + + const folderPath = relativePath(Uri.file(root), folder); + if (folderPath === undefined) { + return undefined; + } + + for (const [projectId, { dir }] of Object.entries(manifest.projects)) { + if (folderPath === dir || folderPath.startsWith(`${dir}/`)) { + return { projectId, projectName: dir.slice(dir.lastIndexOf('/') + 1) }; + } + } + + return undefined; + } + + private async publish(folder: Uri, settings: PublishAppSettings, token: string, force: boolean): Promise { + const syncRoot = await findSyncManifestRoot(getFilePath(folder)).catch((error) => { + // publishApp looks up the same sync root and reports why it is unusable. + logger.debug('Could not look up the sync root of the app folder', error); + + return undefined; + }); + + let lease: Disposable | undefined; + if (syncRoot) { + lease = this.operationLock.tryAcquire(); + if (!lease) { + void window.showInformationMessage(DeepnoteCloud.cloudOperationInProgress); + + return; + } + } + + let mirrorIncompleteRoot: string | undefined; + let outcome: { result: PublishAppResult } | { error: unknown }; + try { + const result = await window.withProgress( + { location: ProgressLocation.Notification, title: DeepnoteCloud.publishing(basename(folder)) }, + (progress) => { + const onEvent = this.progressReporter(progress, (root) => (mirrorIncompleteRoot = root)); + + return this.runPublishApp({ + dir: getFilePath(folder), + projectId: settings.projectId, + baseUrl: getApiEndpoint(folder), + token, + apiAccess: settings.apiAccess, + force, + onEvent + }); + } + ); + outcome = { result }; + } catch (error) { + outcome = { error }; + } finally { + lease?.dispose(); + } + + if ('error' in outcome) { + if (outcome.error instanceof PublishDivergedError) { + await this.resolveDivergence(folder, settings, token, outcome.error); + } else { + void showCloudError(DeepnoteCloud.publishFailed, outcome.error, folder); + } + + return; + } + + const { result } = outcome; + if (result.errors.length > 0) { + void this.showPartialFailure(result.errors.length); + } else if (result.appUrl) { + void showPublishedAppUrl(DeepnoteCloud.publishSucceeded(result.appUrl), result.appUrl); + } + if (mirrorIncompleteRoot) { + void window.showWarningMessage(DeepnoteCloud.publishMirrorIncomplete(mirrorIncompleteRoot)); + } + } + + private async publishFromCommand(arg: Uri | undefined): Promise { + const folder = arg ?? (await this.chooseFolder()); + if (!folder) { + return; + } + + const token = await this.apiTokenService.getOrPromptToken(folder); + if (!token) { + return; + } + + const saved = this.savedSettings()[folder.toString()]; + let settings: PublishAppSettings | undefined; + if (saved) { + const choice = await window.showInformationMessage( + DeepnoteCloud.publishSavedPrompt(basename(folder), saved.projectName), + { + modal: true, + detail: DeepnoteCloud.publishSavedDetail(saved.apiAccess ? DeepnoteCloud.on : DeepnoteCloud.off) + }, + DeepnoteCloud.publish, + DeepnoteCloud.changeSettings + ); + if (choice === DeepnoteCloud.publish) { + settings = saved; + } else if (choice === DeepnoteCloud.changeSettings) { + settings = await this.collectSettings(folder, token, saved); + } + } else { + settings = await this.collectSettings(folder, token); + } + if (!settings) { + return; + } + + await this.publish(folder, settings, token, false); + } + + private progressReporter( + progress: Progress<{ message?: string; increment?: number }>, + onMirrorIncomplete: (syncRoot: string) => void + ): (event: PublishAppEvent) => void { + let total = 0; + let uploaded = 0; + + return (event) => { + switch (event.kind) { + case 'publishing': + total = event.fileCount; + + return; + case 'file-uploaded': + uploaded++; + progress.report({ + message: DeepnoteCloud.publishProgress(uploaded, total), + increment: 100 / total + }); + + return; + case 'operation-failed': + logger.warn(`${event.operation} failed for ${event.path}: ${event.message}`); + + return; + case 'mirror-incomplete': + onMirrorIncomplete(event.syncRoot); + for (const failure of event.failures) { + logger.warn(`Could not update the synced copy of ${failure}`); + } + + return; + default: + logger.debug('Deepnote app publish event', event); + } + }; + } + + private async resolveDivergence( + folder: Uri, + settings: PublishAppSettings, + token: string, + error: PublishDivergedError + ): Promise { + const choice = await window.showWarningMessage( + DeepnoteCloud.publishDiverged(error.paths.length, listPaths(error.paths)), + { modal: true, detail: DeepnoteCloud.publishDivergedDetail }, + DeepnoteCloud.syncFirst, + DeepnoteCloud.force + ); + if (choice === DeepnoteCloud.force) { + await this.publish(folder, settings, token, true); + + return; + } + if (choice !== DeepnoteCloud.syncFirst) { + return; + } + + const syncRoot = Uri.file(error.syncRoot); + const target: SyncWithDeepnoteCloudTarget = { root: syncRoot, allFiles: true }; + const result = await commands.executeCommand( + Commands.SyncWithDeepnoteCloud, + target + ); + if (!result) { + return; + } + + // Only a download writes Deepnote's versions locally; a push of this project uploads local files instead. + const project = result.projects.find((outcome) => outcome.projectId === settings.projectId); + if (project && (project.filesDownloaded ?? 0) > 0) { + const appFiles = Uri.joinPath(syncRoot, projectFilesDir(project.path), PROJECT_STATIC_ROOT); + void window.showInformationMessage(DeepnoteCloud.publishSyncedChanges(workspace.asRelativePath(appFiles))); + + return; + } + + // Notifications render `[label](command:...)` as runnable links, and this name comes from the Deepnote API. + const projectName = settings.projectName.replace(/\]\(/g, '] ('); + void window.showWarningMessage(DeepnoteCloud.publishSyncMissedAppFiles(projectName)); + } + + private savedSettings(): Record { + return this.extensionContext.workspaceState.get>(SETTINGS_STATE_KEY, {}); + } + + private async showPartialFailure(count: number): Promise { + const choice = await window.showWarningMessage( + DeepnoteCloud.publishPartiallyFailed(count), + DeepnoteCloud.showLog + ); + if (choice === DeepnoteCloud.showLog) { + await commands.executeCommand(Commands.ViewJupyterOutput); + } + } +} diff --git a/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.unit.test.ts b/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.unit.test.ts new file mode 100644 index 000000000..791b4044f --- /dev/null +++ b/src/notebooks/deepnote/cloud/publishAppCommandHandler.node.unit.test.ts @@ -0,0 +1,680 @@ +import type { SyncProject } from '@deepnote/cloud'; +import { + findSyncManifestRoot, + PublishDivergedError, + PublishError, + type ProjectSyncOutcome, + type PublishAppOptions, + type PublishAppResult, + type WorkspaceSyncResult +} from '@deepnote/cloud-sync'; +import { assert } from 'chai'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as sinon from 'sinon'; +import { anything, capture, instance, mock, verify, when } from 'ts-mockito'; +import { MessageItem, QuickPickItem, Uri, WorkspaceFolder } from 'vscode'; + +import { Commands } from '../../../platform/common/constants'; +import { IExtensionContext } from '../../../platform/common/types'; +import { createDeferred } from '../../../platform/common/utils/async'; +import * as path from '../../../platform/vscode-path/path'; +import { MockMemento } from '../../../test/mocks/mementos'; +import { mockedVSCodeNamespaces, resetVSCodeMocks } from '../../../test/vscode-mock'; +import { DeepnoteCloudOperationLock } from './deepnoteCloudOperationLock'; +import { PublishAppCommandHandler } from './publishAppCommandHandler.node'; +import { IDeepnoteApiTokenService } from './types'; + +class TestableHandler extends PublishAppCommandHandler { + public readonly listProjectsStub = sinon.stub<[string, string], Promise>(); + public readonly runPublishAppStub = sinon.stub<[PublishAppOptions], Promise>(); + + protected override listProjects(baseUrl: string, token: string): Promise { + return this.listProjectsStub(baseUrl, token); + } + + protected override runPublishApp(options: PublishAppOptions): Promise { + return this.runPublishAppStub(options); + } +} + +type Severity = 'Information' | 'Warning' | 'Error'; +type Answer = string | undefined | (() => string | undefined); + +const PROJECTS: SyncProject[] = [ + { id: 'p-ops', name: 'Ops', folder: null }, + { + id: 'p-sales', + name: 'Sales', + folder: { + id: 'f-q3', + name: 'Q3', + path: [ + { id: 'f-team', name: 'Team' }, + { id: 'f-q3', name: 'Q3' } + ] + } + }, + { id: 'p-alpha', name: 'Alpha' } +]; + +const API_ACCESS_ON_DETAIL = + 'The app calls Deepnote with a 15-minute, viewer-scoped token that can read notebook inputs and start runs. Never your personal token.'; +const CHOOSE_PROJECT = 'Choose the Deepnote project to publish to'; +const CHOOSE_API_ACCESS = 'Should the app be able to call the Deepnote API?'; +const DIVERGED = '1 app file(s) changed in Deepnote since the last sync: _deepnote_static/a.js'; +const MISSED_APP_FILES = + "The sync did not download Deepnote's changed app files for Sales, and nothing was published. Fix any errors or skipped conflicts the sync reported, then publish again and choose Sync First."; + +function published(overrides: Partial = {}): PublishAppResult { + return { + projectId: 'p', + targetPath: '_deepnote_static', + totalFiles: 1, + uploaded: 1, + pruned: 0, + errors: [], + mirrorUpdated: false, + ...overrides + }; +} + +suite('PublishAppCommandHandler', () => { + let tmp: string; + let appFolder: string; + let syncedAppFolder: string; + let handler: TestableHandler; + let tokenService: IDeepnoteApiTokenService; + let lock: DeepnoteCloudOperationLock; + let workspaceState: MockMemento; + let commandHandlers: Map Promise>; + let shown: { severity: Severity; args: unknown[] }[]; + let answers: Map; + let picks: Map; + let quickPicks: { items: QuickPickItem[]; placeHolder: string }[]; + let syncResult: WorkspaceSyncResult | undefined; + let syncTargets: unknown[]; + + const win = () => mockedVSCodeNamespaces.window; + + function messages(severity: Severity): unknown[] { + return shown.filter((message) => message.severity === severity).map((message) => message.args[0]); + } + + function respond([message, ...items]: unknown[]): unknown { + const answer = answers.get(message as string); + const choice = typeof answer === 'function' ? answer() : answer; + + return items.find((item) => item === choice || (item as MessageItem | undefined)?.title === choice); + } + + function quickPick(placeHolder: string): QuickPickItem[] { + const found = quickPicks.find((entry) => entry.placeHolder === placeHolder); + assert.isDefined(found, `no quick pick "${placeHolder}" was shown`); + + return found!.items; + } + + function publishCalls(): PublishAppOptions[] { + return handler.runPublishAppStub.getCalls().map((call) => call.args[0]); + } + + function publish(folder?: string): Promise { + return commandHandlers.get(Commands.PublishApp)!(folder === undefined ? undefined : Uri.file(folder)); + } + + function saveSettings(folder: string, settings: { projectId: string; projectName: string; apiAccess: boolean }) { + return workspaceState.update('deepnote.publishApp.folderSettings', { [Uri.file(folder).toString()]: settings }); + } + + function syncFirstWith(projects: ProjectSyncOutcome[], success = true): void { + handler.runPublishAppStub + .onFirstCall() + .rejects(new PublishDivergedError(path.join(tmp, 'root'), ['_deepnote_static/a.js'])); + answers.set(DIVERGED, 'Sync First'); + syncResult = { success, root: path.join(tmp, 'root'), dryRun: false, projects, untrackedFiles: [] }; + } + + setup(async () => { + resetVSCodeMocks(); + tmp = Uri.file(await fs.promises.mkdtemp(path.join(os.tmpdir(), 'deepnote-publish-'))).fsPath; + assert.isUndefined( + await findSyncManifestRoot(tmp), + 'A .deepnote-sync.json above the temp dir would make every folder synced' + ); + appFolder = path.join(tmp, 'app'); + syncedAppFolder = path.join(tmp, 'root', 'Team', 'Sales', 'app'); + await fs.promises.mkdir(syncedAppFolder, { recursive: true }); + await fs.promises.writeFile( + path.join(tmp, 'root', '.deepnote-sync.json'), + JSON.stringify({ version: 1, projects: { p1: { dir: 'Team/Sales', notebooks: [], contentHash: 'x' } } }) + ); + + const folder: WorkspaceFolder = { uri: Uri.file(tmp), name: 'tmp', index: 0 }; + when(mockedVSCodeNamespaces.workspace.workspaceFolders).thenReturn([folder]); + when(mockedVSCodeNamespaces.workspace.asRelativePath(anything())).thenCall((uri: Uri) => + path.relative(tmp, uri.fsPath).split(path.sep).join('/') + ); + when(mockedVSCodeNamespaces.workspace.getConfiguration('deepnote', anything())).thenReturn({ + get: (_key: string, fallback: unknown) => fallback + } as any); + + shown = []; + answers = new Map(); + for (const severity of ['Information', 'Warning', 'Error'] as const) { + for (let arity = 1; arity <= 4; arity++) { + const matchers = Array.from({ length: arity }, () => anything()); + when((win() as any)[`show${severity}Message`](...matchers)).thenCall(async (...args: unknown[]) => { + shown.push({ severity, args }); + + return respond(args); + }); + } + } + + picks = new Map([ + [CHOOSE_PROJECT, 'Sales'], + [CHOOSE_API_ACCESS, 'API access on'] + ]); + quickPicks = []; + when(win().showQuickPick(anything(), anything())).thenCall( + async (items: QuickPickItem[], options: { placeHolder: string }) => { + quickPicks.push({ items, placeHolder: options.placeHolder }); + const label = picks.get(options.placeHolder); + + return items.find((item) => item.label === label); + } + ); + + commandHandlers = new Map(); + when(mockedVSCodeNamespaces.commands.registerCommand(anything(), anything())).thenCall( + (id: string, commandHandler: (...args: unknown[]) => Promise) => { + commandHandlers.set(id, commandHandler); + + return { dispose: () => undefined }; + } + ); + syncResult = undefined; + syncTargets = []; + when(mockedVSCodeNamespaces.commands.executeCommand(Commands.SyncWithDeepnoteCloud, anything())).thenCall( + async (_command: string, target: unknown) => { + syncTargets.push(target); + + return syncResult; + } + ); + + tokenService = mock(); + when(tokenService.getOrPromptToken(anything())).thenResolve('tok'); + + lock = new DeepnoteCloudOperationLock(); + workspaceState = new MockMemento(); + handler = new TestableHandler(instance(tokenService), lock, { + subscriptions: [], + workspaceState + } as unknown as IExtensionContext); + handler.listProjectsStub.resolves(PROJECTS); + handler.runPublishAppStub.resolves(published()); + handler.activate(); + }); + + teardown(async () => { + resetVSCodeMocks(); + await fs.promises.rm(tmp, { recursive: true, force: true }); + }); + + suite('choosing the folder and the settings', () => { + test('asks for the folder when the command has none', async () => { + when(win().showOpenDialog(anything())).thenCall(async () => [Uri.file(appFolder)]); + + await publish(); + + const dialog = capture(win().showOpenDialog).last()[0]; + assert.deepStrictEqual( + { ...dialog, defaultUri: dialog?.defaultUri?.fsPath }, + { + canSelectFolders: true, + canSelectFiles: false, + canSelectMany: false, + defaultUri: Uri.file(tmp).fsPath, + title: 'Choose the app folder to publish', + openLabel: 'Publish' + } + ); + assert.strictEqual(publishCalls()[0].dir, appFolder); + }); + + test('does nothing when the folder dialog is dismissed', async () => { + await publish(); + + verify(tokenService.getOrPromptToken(anything())).never(); + assert.isTrue(handler.runPublishAppStub.notCalled); + }); + + test('publishes nothing when no token is available', async () => { + when(tokenService.getOrPromptToken(anything())).thenResolve(undefined); + + await publish(appFolder); + + assert.deepStrictEqual(quickPicks, []); + assert.isTrue(handler.runPublishAppStub.notCalled); + }); + + test('asks for the project and API access on a first publish, saves them and publishes the folder', async () => { + await publish(appFolder); + + assert.strictEqual(handler.listProjectsStub.lastCall.args[0], 'https://api.deepnote.com'); + assert.deepStrictEqual( + quickPick(CHOOSE_PROJECT).map(({ label, description }) => ({ label, description })), + [ + { label: 'Alpha', description: undefined }, + { label: 'Ops', description: undefined }, + { label: 'Sales', description: 'Team / Q3' } + ] + ); + assert.deepStrictEqual( + quickPick(CHOOSE_API_ACCESS).map(({ label, detail }) => ({ label, detail })), + [ + { label: 'API access off', detail: 'The app cannot call the Deepnote API.' }, + { label: 'API access on', detail: API_ACCESS_ON_DETAIL } + ] + ); + const { onEvent, ...options } = publishCalls()[0]; + assert.deepStrictEqual(options, { + dir: appFolder, + projectId: 'p-sales', + baseUrl: 'https://api.deepnote.com', + token: 'tok', + apiAccess: true, + force: false + }); + assert.isFunction(onEvent); + assert.deepStrictEqual(workspaceState.get('deepnote.publishApp.folderSettings'), { + [Uri.file(appFolder).toString()]: { projectId: 'p-sales', projectName: 'Sales', apiAccess: true } + }); + }); + + test('saves nothing and publishes nothing when the API-access pick is dismissed', async () => { + picks.set(CHOOSE_API_ACCESS, undefined); + + await publish(appFolder); + + assert.isUndefined(workspaceState.get('deepnote.publishApp.folderSettings')); + assert.isTrue(handler.runPublishAppStub.notCalled); + }); + + test('reports a failed project listing and publishes nothing', async () => { + handler.listProjectsStub.rejects(new Error('boom')); + + await publish(appFolder); + + assert.deepStrictEqual(messages('Error'), ['Could not load Deepnote projects: boom']); + assert.isTrue(handler.runPublishAppStub.notCalled); + }); + + test('uses the synced project that holds the folder without listing projects', async () => { + await publish(syncedAppFolder); + + assert.isTrue(handler.listProjectsStub.notCalled); + assert.strictEqual(publishCalls()[0].projectId, 'p1'); + assert.deepStrictEqual(workspaceState.get('deepnote.publishApp.folderSettings'), { + [Uri.file(syncedAppFolder).toString()]: { projectId: 'p1', projectName: 'Sales', apiAccess: true } + }); + }); + + test('lists the projects for a folder in Team/SalesArchive, beside the synced Team/Sales', async () => { + const besideSynced = path.join(tmp, 'root', 'Team', 'SalesArchive', 'app'); + await fs.promises.mkdir(besideSynced, { recursive: true }); + + await publish(besideSynced); + + assert.isTrue(handler.listProjectsStub.calledOnce); + assert.strictEqual(publishCalls()[0].projectId, 'p-sales'); + }); + + test('lists the projects when the sync manifest above the folder cannot be read', async () => { + await fs.promises.writeFile(path.join(tmp, 'root', '.deepnote-sync.json'), '{'); + + await publish(syncedAppFolder); + + assert.isTrue(handler.listProjectsStub.calledOnce); + assert.strictEqual(publishCalls()[0].projectId, 'p-sales'); + }); + + test('republishes with the saved settings after one confirmation', async () => { + await saveSettings(appFolder, { projectId: 'p-sales', projectName: 'Sales', apiAccess: true }); + answers.set('Publish app to Sales?', 'Publish'); + + await publish(appFolder); + + assert.deepStrictEqual(shown[0].args, [ + 'Publish app to Sales?', + { modal: true, detail: 'API access: on.' }, + 'Publish', + 'Change Settings…' + ]); + assert.deepStrictEqual(quickPicks, []); + assert.isTrue(handler.runPublishAppStub.calledOnce); + assert.strictEqual(publishCalls()[0].projectId, 'p-sales'); + }); + + test('lists the current choices first when changing the saved settings', async () => { + await saveSettings(appFolder, { projectId: 'p-sales', projectName: 'Sales', apiAccess: true }); + answers.set('Publish app to Sales?', 'Change Settings…'); + picks.set(CHOOSE_PROJECT, 'Ops'); + picks.set(CHOOSE_API_ACCESS, 'API access off'); + + await publish(appFolder); + + assert.deepStrictEqual( + quickPick(CHOOSE_PROJECT).map(({ label, detail }) => ({ label, detail })), + [ + { label: 'Sales', detail: 'Current choice' }, + { label: 'Alpha', detail: undefined }, + { label: 'Ops', detail: undefined } + ] + ); + assert.deepStrictEqual( + quickPick(CHOOSE_API_ACCESS).map(({ label, detail }) => ({ label, detail })), + [ + { label: 'API access on', detail: `Current choice. ${API_ACCESS_ON_DETAIL}` }, + { label: 'API access off', detail: 'The app cannot call the Deepnote API.' } + ] + ); + assert.deepStrictEqual(workspaceState.get('deepnote.publishApp.folderSettings'), { + [Uri.file(appFolder).toString()]: { projectId: 'p-ops', projectName: 'Ops', apiAccess: false } + }); + assert.include(publishCalls()[0], { projectId: 'p-ops', apiAccess: false }); + }); + + test('publishes nothing when the saved-settings confirmation is dismissed', async () => { + await saveSettings(appFolder, { projectId: 'p-sales', projectName: 'Sales', apiAccess: false }); + + await publish(appFolder); + + assert.deepStrictEqual(quickPicks, []); + assert.isTrue(handler.runPublishAppStub.notCalled); + }); + }); + + suite('one operation at a time', () => { + test('publishes a folder outside any sync root while the lock is held', async () => { + const lease = lock.tryAcquire()!; + + await publish(appFolder); + + assert.isTrue(handler.runPublishAppStub.calledOnce); + lease.dispose(); + }); + + test('refuses to publish into a synced folder while the lock is held', async () => { + const lease = lock.tryAcquire()!; + + await publish(syncedAppFolder); + + assert.deepStrictEqual(messages('Information'), [ + 'A sync or app publish with Deepnote Cloud is already in progress.' + ]); + assert.isTrue(handler.runPublishAppStub.notCalled); + lease.dispose(); + }); + + test('holds the lock while publishing into a synced folder, and releases it once the publish settles', async () => { + let lockHeldDuringPublish = false; + handler.runPublishAppStub.callsFake(async () => { + const lease = lock.tryAcquire(); + lockHeldDuringPublish = lease === undefined; + lease?.dispose(); + + return published(); + }); + + await publish(syncedAppFolder); + + assert.isTrue(lockHeldDuringPublish, 'the lock must be held while publishing'); + assert.isDefined(lock.tryAcquire(), 'the lock must be released once the publish settles'); + }); + }); + + suite('results', () => { + test('reports upload progress', async () => { + const reports: unknown[] = []; + when(win().withProgress(anything(), anything())).thenCall((_options, task) => + task({ report: (value: unknown) => reports.push(value) }, undefined) + ); + handler.runPublishAppStub.callsFake(async ({ onEvent }) => { + onEvent?.({ kind: 'publishing', fileCount: 2, targetPath: '_deepnote_static', projectId: 'p' }); + onEvent?.({ kind: 'file-uploaded', path: 'index.html' }); + onEvent?.({ kind: 'file-uploaded', path: 'main.js' }); + + return published(); + }); + + await publish(appFolder); + + assert.deepStrictEqual(reports, [ + { message: '1 of 2 files uploaded', increment: 50 }, + { message: '2 of 2 files uploaded', increment: 50 } + ]); + }); + + test('shows the app URL with Open and Copy, and Copy copies it', async () => { + const url = 'https://deepnote.com/app/x'; + handler.runPublishAppStub.resolves(published({ appUrl: url })); + const copied = createDeferred(); + when(mockedVSCodeNamespaces.env.clipboard).thenReturn({ + writeText: async (text: string) => copied.resolve(text) + } as any); + answers.set(`App published: ${url}`, 'Copy'); + + await publish(appFolder); + + assert.deepStrictEqual(shown[shown.length - 1].args, [`App published: ${url}`, 'Open', 'Copy']); + assert.strictEqual(await copied.promise, url); + verify(win().setStatusBarMessage('App URL copied', 3000)).once(); + }); + + test('opens the app URL in the browser', async () => { + const url = 'https://deepnote.com/app/x'; + handler.runPublishAppStub.resolves(published({ appUrl: url })); + const opened = createDeferred(); + when(mockedVSCodeNamespaces.env.openExternal(anything())).thenCall(async (uri: Uri) => { + opened.resolve(uri); + + return true; + }); + answers.set(`App published: ${url}`, 'Open'); + + await publish(appFolder); + + assert.strictEqual((await opened.promise).toString(), url); + }); + + test('reports failed file operations with Show Log and no URL', async () => { + handler.runPublishAppStub.resolves( + published({ errors: [{ path: 'index.html', message: 'upload failed' }], appUrl: undefined }) + ); + answers.set('1 file operation(s) failed, so the app was not shared. See the log for details.', 'Show Log'); + const logShown = createDeferred(); + when(mockedVSCodeNamespaces.commands.executeCommand(Commands.ViewJupyterOutput)).thenCall(async () => + logShown.resolve() + ); + + await publish(appFolder); + + assert.deepStrictEqual(shown[shown.length - 1], { + severity: 'Warning', + args: ['1 file operation(s) failed, so the app was not shared. See the log for details.', 'Show Log'] + }); + assert.deepStrictEqual(messages('Information'), []); + await logShown.promise; + }); + + test('warns after the URL when the synced copy could not be fully updated', async () => { + handler.runPublishAppStub.callsFake(async ({ onEvent }) => { + onEvent?.({ kind: 'mirror-incomplete', syncRoot: '/sync', failures: ['index.html — EACCES'] }); + + return published({ appUrl: 'https://deepnote.com/app/x' }); + }); + + await publish(appFolder); + + assert.deepStrictEqual( + shown.map(({ severity, args }) => [severity, args[0]]), + [ + ['Information', 'App published: https://deepnote.com/app/x'], + [ + 'Warning', + 'The app was published, but the synced copy in /sync could not be fully updated. Sync again to refresh it.' + ] + ] + ); + }); + + test('reports a refused publish', async () => { + handler.runPublishAppStub.rejects(new PublishError('invalid-input', 'Refusing to publish ".env"')); + + await publish(appFolder); + + assert.deepStrictEqual(messages('Error'), ['Publishing the app failed: Refusing to publish ".env"']); + assert.deepStrictEqual(messages('Information'), []); + }); + }); + + suite('app files changed in Deepnote', () => { + setup(() => { + handler.runPublishAppStub + .onFirstCall() + .rejects(new PublishDivergedError(path.join(tmp, 'root'), ['_deepnote_static/a.js'])); + }); + + test('asks once the lock is released, and Force publishes again with force', async () => { + let lockFreeDuringModal = false; + answers.set(DIVERGED, () => { + const lease = lock.tryAcquire(); + lockFreeDuringModal = lease !== undefined; + lease?.dispose(); + + return 'Force'; + }); + + await publish(syncedAppFolder); + + assert.deepStrictEqual(shown[0], { + severity: 'Warning', + args: [ + DIVERGED, + { + modal: true, + detail: "Sync First syncs the whole synced folder, including the working files of every project, so you can merge Deepnote's versions into your app. Force overwrites them in Deepnote." + }, + 'Sync First', + 'Force' + ] + }); + assert.isTrue(lockFreeDuringModal); + assert.deepStrictEqual( + publishCalls().map(({ force }) => force), + [false, true] + ); + }); + + for (const { name, count, message } of [ + { + name: 'lists all of ten changed paths', + count: 10, + message: + '10 app file(s) changed in Deepnote since the last sync: f0, f1, f2, f3, f4, f5, f6, f7, f8, f9' + }, + { + name: 'lists ten of twelve changed paths and counts the rest', + count: 12, + message: + '12 app file(s) changed in Deepnote since the last sync: f0, f1, f2, f3, f4, f5, f6, f7, f8, f9 …and 2 more' + } + ]) { + test(name, async () => { + const paths = Array.from({ length: count }, (_, i) => `f${i}`); + handler.runPublishAppStub + .onFirstCall() + .rejects(new PublishDivergedError(path.join(tmp, 'root'), paths)); + + await publish(syncedAppFolder); + + assert.strictEqual(messages('Warning')[0], message); + }); + } + + test('leaves everything as it is when the question is dismissed', async () => { + await publish(syncedAppFolder); + + assert.isTrue(handler.runPublishAppStub.calledOnce); + assert.deepStrictEqual(syncTargets, []); + }); + + test('Sync First syncs the root with working files and names where the app files landed', async () => { + syncFirstWith([ + { projectId: 'p1', name: 'Sales', path: 'Team/Sales', action: 'unchanged', filesDownloaded: 1 } + ]); + + await publish(syncedAppFolder); + + const [target] = syncTargets as { root: Uri; allFiles: boolean }[]; + assert.deepStrictEqual( + { ...target, root: target.root.fsPath }, + { root: path.join(tmp, 'root'), allFiles: true } + ); + assert.isTrue(handler.runPublishAppStub.calledOnce); + assert.deepStrictEqual(messages('Information'), [ + "Deepnote's versions of the changed app files are now in root/Team/Sales/.files/_deepnote_static. Merge them into your app, then publish again." + ]); + }); + + test('Sync First warns when the project failed, even though another project downloaded files', async () => { + syncFirstWith( + [ + { projectId: 'p1', name: 'Sales', path: 'Team/Sales', action: 'error', detail: 'export failed' }, + { projectId: 'p2', name: 'Ops', path: 'Ops', action: 'pulled', filesDownloaded: 2 } + ], + false + ); + + await publish(syncedAppFolder); + + assert.deepStrictEqual(messages('Warning').slice(1), [MISSED_APP_FILES]); + assert.deepStrictEqual(messages('Information'), []); + }); + + test('Sync First warns when the sync succeeded but pushed the project instead of downloading it', async () => { + syncFirstWith([{ projectId: 'p1', name: 'Sales', path: 'Team/Sales', action: 'pushed', filesUploaded: 0 }]); + + await publish(syncedAppFolder); + + assert.deepStrictEqual(messages('Warning').slice(1), [MISSED_APP_FILES]); + }); + + test('Sync First keeps a project name from Deepnote from becoming a link in its warning', async () => { + const folder = path.join(tmp, 'root', 'site'); + handler.listProjectsStub.resolves([{ id: 'p-x', name: 'X [Show Log](command:evil)' }]); + picks.set(CHOOSE_PROJECT, 'X [Show Log](command:evil)'); + syncFirstWith([{ projectId: 'p-x', name: 'X', path: 'X', action: 'pushed' }]); + + await publish(folder); + + assert.deepStrictEqual(messages('Warning').slice(1), [ + "The sync did not download Deepnote's changed app files for X [Show Log] (command:evil), and nothing was published. Fix any errors or skipped conflicts the sync reported, then publish again and choose Sync First." + ]); + }); + + test('Sync First shows nothing more when the sync stopped', async () => { + syncFirstWith([]); + syncResult = undefined; + + await publish(syncedAppFolder); + + assert.deepStrictEqual(messages('Warning'), [DIVERGED]); + assert.deepStrictEqual(messages('Information'), []); + assert.isTrue(handler.runPublishAppStub.calledOnce); + }); + }); +}); diff --git a/src/notebooks/deepnote/cloud/publishedAppUrl.node.ts b/src/notebooks/deepnote/cloud/publishedAppUrl.node.ts new file mode 100644 index 000000000..c4061ecd6 --- /dev/null +++ b/src/notebooks/deepnote/cloud/publishedAppUrl.node.ts @@ -0,0 +1,16 @@ +import { env, Uri, window } from 'vscode'; + +import { DeepnoteCloud } from '../../../platform/common/utils/localize'; + +export const COPIED_MESSAGE_TIMEOUT_MS = 3000; + +/** Shows `message` with "Open", which opens `url` in the browser, and "Copy", which copies it to the clipboard. */ +export async function showPublishedAppUrl(message: string, url: string): Promise { + const choice = await window.showInformationMessage(message, DeepnoteCloud.open, DeepnoteCloud.copy); + if (choice === DeepnoteCloud.open) { + await env.openExternal(Uri.parse(url)); + } else if (choice === DeepnoteCloud.copy) { + await env.clipboard.writeText(url); + window.setStatusBarMessage(DeepnoteCloud.copiedToClipboard, COPIED_MESSAGE_TIMEOUT_MS); + } +} diff --git a/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.ts b/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.ts index ffab35688..d7a6f638e 100644 --- a/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.ts +++ b/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.ts @@ -33,7 +33,7 @@ import { logger } from '../../../platform/logging'; import { dirname, isEqual } from '../../../platform/vscode-path/resources'; import { getApiEndpoint } from '../importClient.node'; import { showCloudError } from './cloudErrors.node'; -import { IDeepnoteApiTokenService } from './types'; +import { IDeepnoteApiTokenService, IDeepnoteCloudOperationLock, SyncWithDeepnoteCloudTarget } from './types'; interface RootPickItem extends QuickPickItem { root?: Uri; @@ -54,6 +54,12 @@ function unsavedDeepnoteDocuments(): (NotebookDocument | TextDocument)[] { ]; } +function asSyncTarget(arg: unknown): SyncWithDeepnoteCloudTarget | undefined { + const candidate = arg as Partial | undefined; + + return candidate?.root instanceof Uri ? { root: candidate.root, allFiles: candidate.allFiles === true } : undefined; +} + function logSyncEvent(event: SyncEvent): void { switch (event.kind) { case 'project-outcome': { @@ -83,16 +89,17 @@ async function askOverride(question: string, overrideLabel: string): Promise this.syncWithDeepnoteCloud()) + commands.registerCommand(Commands.SyncWithDeepnoteCloud, (target?: unknown) => + this.syncWithDeepnoteCloud(asSyncTarget(target)) + ) ); } @@ -118,11 +125,14 @@ export class SyncWithDeepnoteCloudCommandHandler implements IExtensionSyncActiva DeepnoteCloud.conflictUntrackedLocalDirectory(conflict.projectName, conflict.projectDir), DeepnoteCloud.conflictUntrackedLocalDirectoryOverride ); - // Only `deleteMissingNotebooks` and `allFiles` produce these, and this command sets neither. + // Sync raises this only with `deleteMissingNotebooks`, which is never set. case 'empty-local-directory': - case 'working-files-changed': logger.warn(`Skipped an unexpected ${conflict.kind} conflict for "${conflict.projectName}"`); + return Promise.resolve('skip'); + // Sync raises this only with `allFiles`, where `override` would upload local files over Deepnote's. On a + // skip, sync warns which files it kept. + case 'working-files-changed': return Promise.resolve('skip'); } } @@ -165,8 +175,8 @@ export class SyncWithDeepnoteCloudCommandHandler implements IExtensionSyncActiva return root ? Uri.file(root) : folder; } - private async chooseSyncTarget(): Promise<{ root: Uri; isFirstSync: boolean } | undefined> { - const root = await this.chooseRoot(); + private async chooseSyncTarget(requestedRoot?: Uri): Promise<{ root: Uri; isFirstSync: boolean } | undefined> { + const root = requestedRoot ?? (await this.chooseRoot()); if (!root) { return undefined; } @@ -268,10 +278,12 @@ export class SyncWithDeepnoteCloudCommandHandler implements IExtensionSyncActiva } } - private async syncChosenFolder(): Promise { + private async syncChosenFolder( + requested: SyncWithDeepnoteCloudTarget | undefined + ): Promise { let target: { root: Uri; isFirstSync: boolean } | undefined; try { - target = await this.chooseSyncTarget(); + target = await this.chooseSyncTarget(requested?.root); } catch (error) { void showCloudError(DeepnoteCloud.syncFailed, error); @@ -304,6 +316,7 @@ export class SyncWithDeepnoteCloudCommandHandler implements IExtensionSyncActiva rootDir: getFilePath(root), baseUrl: getApiEndpoint(root), token, + allFiles: requested?.allFiles === true, onConflict: (conflict) => this.askConflict(conflict), onEvent: logSyncEvent }) @@ -314,22 +327,26 @@ export class SyncWithDeepnoteCloudCommandHandler implements IExtensionSyncActiva return; } - // Not awaited, so a summary left on screen does not hold the in-progress flag. + // Not awaited, so a summary left on screen does not hold the lock. void this.showSummary(result); + + return result; } - private async syncWithDeepnoteCloud(): Promise { - if (this.syncInProgress) { - void window.showInformationMessage(DeepnoteCloud.syncInProgress); + private async syncWithDeepnoteCloud( + target: SyncWithDeepnoteCloudTarget | undefined + ): Promise { + const lease = this.operationLock.tryAcquire(); + if (!lease) { + void window.showInformationMessage(DeepnoteCloud.cloudOperationInProgress); - return; + return undefined; } - this.syncInProgress = true; try { - await this.syncChosenFolder(); + return await this.syncChosenFolder(target); } finally { - this.syncInProgress = false; + lease.dispose(); } } } diff --git a/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.unit.test.ts b/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.unit.test.ts index 805779939..6e871b70e 100644 --- a/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.unit.test.ts +++ b/src/notebooks/deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node.unit.test.ts @@ -31,6 +31,7 @@ import { DeepnoteCloud } from '../../../platform/common/utils/localize'; import { logger } from '../../../platform/logging'; import * as path from '../../../platform/vscode-path/path'; import { mockedVSCodeNamespaces, resetVSCodeMocks } from '../../../test/vscode-mock'; +import { DeepnoteCloudOperationLock } from './deepnoteCloudOperationLock'; import { SyncWithDeepnoteCloudCommandHandler } from './syncWithDeepnoteCloudCommandHandler.node'; import { IDeepnoteApiTokenService } from './types'; @@ -63,7 +64,8 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { let ws: string; let handler: TestableHandler; let tokenService: IDeepnoteApiTokenService; - let commandHandlers: Map Promise>; + let lock: DeepnoteCloudOperationLock; + let commandHandlers: Map Promise>; let manifests: { file: string; hidden: boolean }[]; let shown: { severity: Severity; args: unknown[] }[]; let answers: Map>; @@ -135,8 +137,8 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { when(win().showOpenDialog(anything())).thenCall(async () => [Uri.file(folder)]); } - function sync(): Promise { - return commandHandlers.get(Commands.SyncWithDeepnoteCloud)!(); + function sync(...args: unknown[]): Promise { + return commandHandlers.get(Commands.SyncWithDeepnoteCloud)!(...args); } function syncOptions(): WorkspaceSyncOptions { @@ -193,7 +195,7 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { commandHandlers = new Map(); when(mockedVSCodeNamespaces.commands.registerCommand(anything(), anything())).thenCall( - (id: string, commandHandler: () => Promise) => { + (id: string, commandHandler: (...args: unknown[]) => Promise) => { commandHandlers.set(id, commandHandler); return { dispose: () => undefined }; @@ -203,7 +205,10 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { tokenService = mock(); when(tokenService.getOrPromptToken(anything())).thenResolve('tok'); - handler = new TestableHandler(instance(tokenService), { subscriptions: [] } as unknown as IExtensionContext); + lock = new DeepnoteCloudOperationLock(); + handler = new TestableHandler(instance(tokenService), lock, { + subscriptions: [] + } as unknown as IExtensionContext); handler.runSyncWorkspaceStub.resolves(EMPTY_RESULT); handler.activate(); }); @@ -237,7 +242,8 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { assert.deepStrictEqual(rest, { rootDir: path.join(ws, 'b'), baseUrl: 'https://api.example.test', - token: 'tok' + token: 'tok', + allFiles: false }); assert.isFunction(onConflict); assert.isFunction(onEvent); @@ -319,6 +325,34 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { }); }); + suite('syncing a requested folder', () => { + test('syncs the requested folder with working files, without asking for a folder', async () => { + await addManifest('a'); + const result: WorkspaceSyncResult = { ...EMPTY_RESULT, root: path.join(ws, 'a') }; + handler.runSyncWorkspaceStub.resolves(result); + + const resolved = await sync({ root: Uri.file(path.join(ws, 'a')), allFiles: true }); + + verify(win().showQuickPick(anything(), anything())).never(); + verify(win().showOpenDialog(anything())).never(); + assert.strictEqual(syncOptions().rootDir, path.join(ws, 'a')); + assert.isTrue(syncOptions().allFiles); + assert.strictEqual(resolved, result); + }); + + test('refuses a requested folder outside the workspace', async () => { + const outside = path.join(tmp, 'outside'); + + const resolved = await sync({ root: Uri.file(outside), allFiles: true }); + + assert.deepStrictEqual(messages('Warning'), [ + `${outside} is outside the open workspace. Open it as a workspace folder to sync it with Deepnote Cloud.` + ]); + verifyNoTokenRequested(); + assert.isUndefined(resolved); + }); + }); + suite('refused folders', () => { test('refuses a folder outside the workspace', async () => { const outside = path.join(tmp, 'outside'); @@ -747,7 +781,9 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { await started.promise; await sync(); - assert.deepStrictEqual(messages('Information'), ['A sync with Deepnote Cloud is already in progress.']); + assert.deepStrictEqual(messages('Information'), [ + 'A sync or app publish with Deepnote Cloud is already in progress.' + ]); verify(win().showQuickPick(anything(), anything())).once(); assert.isTrue(handler.runSyncWorkspaceStub.calledOnce); @@ -755,6 +791,23 @@ suite('SyncWithDeepnoteCloudCommandHandler', () => { await first; }); + test('refuses to sync while another operation holds the lock, and syncs once it is released', async () => { + const lease = lock.tryAcquire()!; + + await sync(); + + assert.deepStrictEqual(messages('Information'), [ + 'A sync or app publish with Deepnote Cloud is already in progress.' + ]); + verify(win().showQuickPick(anything(), anything())).never(); + assert.isTrue(handler.runSyncWorkspaceStub.notCalled); + + lease.dispose(); + await sync(); + + assert.isTrue(handler.runSyncWorkspaceStub.calledOnce); + }); + test('does not wait for the summary to be dismissed', async () => { answer( 'Sync finished: 0 pulled, 0 pushed, 0 unchanged, 0 skipped, 0 failed.', diff --git a/src/notebooks/deepnote/cloud/types.ts b/src/notebooks/deepnote/cloud/types.ts index b655d4c3f..b371631b8 100644 --- a/src/notebooks/deepnote/cloud/types.ts +++ b/src/notebooks/deepnote/cloud/types.ts @@ -1,4 +1,4 @@ -import { Uri } from 'vscode'; +import { Disposable, Uri } from 'vscode'; export const IDeepnoteApiTokenService = Symbol('IDeepnoteApiTokenService'); export interface IDeepnoteApiTokenService { @@ -20,3 +20,19 @@ export interface IDeepnoteApiTokenService { */ getOrPromptToken(resource: Uri): Promise; } + +export const IDeepnoteCloudOperationLock = Symbol('IDeepnoteCloudOperationLock'); +export interface IDeepnoteCloudOperationLock { + /** + * Marks a sync, or a publish into a synced folder, as running in this window. `undefined` when one already is. + * Disposing the result releases the lock. + */ + tryAcquire(): Disposable | undefined; +} + +/** The argument of `deepnote.syncWithDeepnoteCloud` that syncs `root` without asking for a folder. */ +export interface SyncWithDeepnoteCloudTarget { + root: Uri; + /** Also sync each project's working files, such as app files. */ + allFiles: boolean; +} diff --git a/src/notebooks/serviceRegistry.node.ts b/src/notebooks/serviceRegistry.node.ts index 440f23ce2..54b2fe0f2 100644 --- a/src/notebooks/serviceRegistry.node.ts +++ b/src/notebooks/serviceRegistry.node.ts @@ -100,8 +100,10 @@ import { OrphanedEphemeralCellCleaner } from './deepnote/orphanedEphemeralCellCl import { DeepnoteNewCellLanguageService } from './deepnote/deepnoteNewCellLanguageService'; import { DeepnoteCellCopyHandler } from './deepnote/deepnoteCellCopyHandler'; import { DeepnoteApiTokenService } from './deepnote/cloud/deepnoteApiTokenService.node'; +import { DeepnoteCloudOperationLock } from './deepnote/cloud/deepnoteCloudOperationLock'; +import { PublishAppCommandHandler } from './deepnote/cloud/publishAppCommandHandler.node'; import { SyncWithDeepnoteCloudCommandHandler } from './deepnote/cloud/syncWithDeepnoteCloudCommandHandler.node'; -import { IDeepnoteApiTokenService } from './deepnote/cloud/types'; +import { IDeepnoteApiTokenService, IDeepnoteCloudOperationLock } from './deepnote/cloud/types'; import { OpenInDeepnoteHandler } from './deepnote/openInDeepnoteHandler.node'; import { IntegrationEnvRefreshHandler } from './deepnote/integrations/integrationEnvRefreshHandler'; import { IntegrationsEnvFileWatcher } from './deepnote/integrations/integrationsEnvFileWatcher.node'; @@ -235,10 +237,15 @@ export function registerTypes(serviceManager: IServiceManager, isDevMode: boolea ); serviceManager.addSingleton(IDeepnoteApiTokenService, DeepnoteApiTokenService); serviceManager.addBinding(IDeepnoteApiTokenService, IExtensionSyncActivationService); + serviceManager.addSingleton(IDeepnoteCloudOperationLock, DeepnoteCloudOperationLock); serviceManager.addSingleton( IExtensionSyncActivationService, SyncWithDeepnoteCloudCommandHandler ); + serviceManager.addSingleton( + IExtensionSyncActivationService, + PublishAppCommandHandler + ); serviceManager.addSingleton( IExtensionSyncActivationService, IntegrationEnvRefreshHandler diff --git a/src/platform/common/constants.ts b/src/platform/common/constants.ts index 836c69ce6..1d96af1cd 100644 --- a/src/platform/common/constants.ts +++ b/src/platform/common/constants.ts @@ -276,6 +276,7 @@ export namespace Commands { export const SetApiToken = 'deepnote.setApiToken'; export const ClearApiToken = 'deepnote.clearApiToken'; export const SyncWithDeepnoteCloud = 'deepnote.syncWithDeepnoteCloud'; + export const PublishApp = 'deepnote.publishApp'; export const ExportAsPythonScript = 'deepnote.exportAsPythonScript'; export const ExportToHTML = 'deepnote.exportToHTML'; export const ExportToPDF = 'deepnote.exportToPDF'; diff --git a/src/platform/common/utils/localize.ts b/src/platform/common/utils/localize.ts index 4cdc19e34..35081d19b 100644 --- a/src/platform/common/utils/localize.ts +++ b/src/platform/common/utils/localize.ts @@ -1314,7 +1314,7 @@ export namespace DeepnoteCloud { export const errorWithDetail = (message: string, detail: string) => l10n.t('{0}: {1}', message, detail); export const syncing = l10n.t('Syncing with Deepnote Cloud…'); export const syncFailed = l10n.t('Sync with Deepnote Cloud failed'); - export const syncInProgress = l10n.t('A sync with Deepnote Cloud is already in progress.'); + export const cloudOperationInProgress = l10n.t('A sync or app publish with Deepnote Cloud is already in progress.'); export const chooseSyncFolder = l10n.t('Choose the folder to sync with Deepnote Cloud'); export const chooseAnotherSyncFolder = l10n.t('Choose another folder…'); export const chooseSyncFolderTitle = l10n.t('Choose a folder for your Deepnote workspace'); @@ -1358,6 +1358,58 @@ export namespace DeepnoteCloud { export const conflictUntrackedLocalDirectory = (project: string, dir: string) => l10n.t('{1} already exists locally and differs from the Deepnote project "{0}".', project, dir); export const conflictUntrackedLocalDirectoryOverride = l10n.t('Replace Local Files with Deepnote Version'); + export const andMore = (count: number) => l10n.t('…and {0} more', count); + export const chooseAppFolderTitle = l10n.t('Choose the app folder to publish'); + export const publish = l10n.t('Publish'); + export const loadingProjects = l10n.t('Loading Deepnote projects…'); + export const loadProjectsFailed = l10n.t('Could not load Deepnote projects'); + export const chooseProject = l10n.t('Choose the Deepnote project to publish to'); + export const currentChoice = l10n.t('Current choice'); + export const apiAccessOff = l10n.t('API access off'); + export const apiAccessOffDetail = l10n.t('The app cannot call the Deepnote API.'); + export const apiAccessOn = l10n.t('API access on'); + export const apiAccessOnDetail = l10n.t( + 'The app calls Deepnote with a 15-minute, viewer-scoped token that can read notebook inputs and start runs. Never your personal token.' + ); + export const chooseApiAccess = l10n.t('Should the app be able to call the Deepnote API?'); + export const publishSavedPrompt = (folder: string, project: string) => + l10n.t('Publish {0} to {1}?', folder, project); + export const publishSavedDetail = (apiAccess: string) => l10n.t('API access: {0}.', apiAccess); + export const on = l10n.t('on'); + export const off = l10n.t('off'); + export const changeSettings = l10n.t('Change Settings…'); + export const publishing = (folder: string) => l10n.t('Publishing {0} to Deepnote…', folder); + export const publishProgress = (uploaded: number, total: number) => + l10n.t('{0} of {1} files uploaded', uploaded, total); + export const publishFailed = l10n.t('Publishing the app failed'); + export const publishPartiallyFailed = (count: number) => + l10n.t('{0} file operation(s) failed, so the app was not shared. See the log for details.', count); + export const publishSucceeded = (url: string) => l10n.t('App published: {0}', url); + export const open = l10n.t('Open'); + export const copy = l10n.t('Copy'); + export const copiedToClipboard = l10n.t('App URL copied'); + export const publishMirrorIncomplete = (syncRoot: string) => + l10n.t( + 'The app was published, but the synced copy in {0} could not be fully updated. Sync again to refresh it.', + syncRoot + ); + export const publishDiverged = (count: number, paths: string) => + l10n.t('{0} app file(s) changed in Deepnote since the last sync: {1}', count, paths); + export const publishDivergedDetail = l10n.t( + "Sync First syncs the whole synced folder, including the working files of every project, so you can merge Deepnote's versions into your app. Force overwrites them in Deepnote." + ); + export const syncFirst = l10n.t('Sync First'); + export const force = l10n.t('Force'); + export const publishSyncedChanges = (folder: string) => + l10n.t( + "Deepnote's versions of the changed app files are now in {0}. Merge them into your app, then publish again.", + folder + ); + export const publishSyncMissedAppFiles = (project: string) => + l10n.t( + "The sync did not download Deepnote's changed app files for {0}, and nothing was published. Fix any errors or skipped conflicts the sync reported, then publish again and choose Sync First.", + project + ); } export namespace SelectInputSettings { diff --git a/test/e2e/helpers/apiToken.ts b/test/e2e/helpers/apiToken.ts new file mode 100644 index 000000000..6dc400a85 --- /dev/null +++ b/test/e2e/helpers/apiToken.ts @@ -0,0 +1,30 @@ +import { InputBox, Workbench } from 'vscode-extension-tester'; + +import { QUICK_PICK_TIMEOUT, WORKBENCH_TIMEOUT } from './constants'; +import { dismissAllNotifications, waitForNotification } from './notifications'; + +export const API_TOKEN = 'e2e_mock_token'; + +/** + * Saves {@link API_TOKEN} with "Deepnote: Set API Token". Resolves with the confirmation message, or throws with the + * error one. + */ +export async function setApiToken(): Promise { + await dismissAllNotifications(); + await new Workbench().executeCommand('Deepnote: Set API Token'); + const input = await InputBox.create(QUICK_PICK_TIMEOUT); + await input.setText(API_TOKEN); + await input.confirm(); + + const outcome = await waitForNotification( + /^(Signed in to the Deepnote workspace|Deepnote API token saved|Deepnote rejected|Could not check|Could not save)/, + WORKBENCH_TIMEOUT, + true + ); + const message = (await outcome?.getMessage()) ?? ''; + if (/^(Deepnote rejected|Could not)/.test(message)) { + throw new Error(message); + } + + return message; +} diff --git a/test/e2e/helpers/index.ts b/test/e2e/helpers/index.ts index 8fff268cc..af546c1bb 100644 --- a/test/e2e/helpers/index.ts +++ b/test/e2e/helpers/index.ts @@ -1,3 +1,4 @@ +export * from './apiToken'; export * from './assertions'; export * from './constants'; export * from './deepnoteTree'; diff --git a/test/e2e/helpers/mockDeepnoteCloud.ts b/test/e2e/helpers/mockDeepnoteCloud.ts index effdee1cf..42a0c3761 100644 --- a/test/e2e/helpers/mockDeepnoteCloud.ts +++ b/test/e2e/helpers/mockDeepnoteCloud.ts @@ -1,11 +1,13 @@ /** - * A minimal Deepnote Cloud: the Public API v2 routes that "Deepnote: Set API Token" and `syncWorkspace` call, - * backed by a directory with one subdirectory per project, named by its id and holding its `.deepnote` files. - * Editing those files is how a test, or a person, changes a project "in Deepnote". Any Bearer token is accepted. + * A minimal Deepnote Cloud: the Public API v2 routes that "Deepnote: Set API Token", `syncWorkspace` and + * `publishApp` call, backed by a directory with one subdirectory per project, named by its id and holding its + * `.deepnote` files. The project's working files, such as a published app under `_deepnote_static/`, live below + * its `files/` subdirectory. Editing those files is how a test, or a person, changes a project "in Deepnote". + * Static-file sharing settings are kept in memory. Any Bearer token is accepted. */ import { deserializeDeepnoteFile, type DeepnoteFile } from '@deepnote/blocks'; -import { sha256 } from '@deepnote/cloud-sync'; +import { isSafeRelativeFilePath, sha256 } from '@deepnote/cloud-sync'; import express, { type Response } from 'express'; import { unzipSync, zipSync } from 'fflate'; import { once } from 'events'; @@ -14,13 +16,16 @@ import type { AddressInfo } from 'net'; import * as os from 'os'; import * as path from 'path'; -import type { components } from './deepnoteCloudApi.generated'; +import type { components, operations } from './deepnoteCloudApi.generated'; type Schemas = components['schemas']; +type UpdateProjectBody = operations['updateProject']['requestBody']['content']['application/json']; const PROJECT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const NOTEBOOK_FILE = /^[^/\\]+\.deepnote$/; const MAX_ARCHIVE_SIZE = '100mb'; +const MAX_FILE_SIZE = '100mb'; +const FILES_DIR = 'files'; export const MOCK_WORKSPACE_NAME = 'Mock Workspace'; @@ -84,6 +89,48 @@ function findImportViolation(projectId: string, projects: DeepnoteFile['project' return undefined; } +/** `publishApp` detects cloud edits by comparing each entry's `size` and `updatedAt`. */ +async function listProjectFiles(filesDir: string): Promise { + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(filesDir, { recursive: true, withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } + + return Promise.all( + entries + .filter((entry) => entry.isFile()) + .map(async (entry): Promise => { + const absolute = path.join(entry.parentPath, entry.name); + const { size, mtime } = await fs.promises.stat(absolute); + + return { + name: entry.name, + path: path.relative(filesDir, absolute).split(path.sep).join('/'), + type: 'file', + size, + updatedAt: mtime.toISOString() + }; + }) + ); +} + +/** Like Deepnote, stores an upload to a taken path at a new unique path instead of overwriting it. */ +function freeFilePath(filesDir: string, filePath: string): string { + const extension = path.posix.extname(filePath); + const stem = filePath.slice(0, filePath.length - extension.length); + let candidate = filePath; + for (let n = 1; fs.existsSync(path.join(filesDir, ...candidate.split('/'))); n++) { + candidate = `${stem} (${n})${extension}`; + } + + return candidate; +} + function sendError(res: Response, status: number, message: string): void { const body: Schemas['ErrorResponse'] = { message }; res.status(status).json(body); @@ -95,6 +142,25 @@ export async function startMockDeepnoteCloud(rootDir?: string): Promise path.join(root, projectId); + const filesDir = (projectId: string) => path.join(projectDir(projectId), FILES_DIR); + const isKnownProject = (projectId: unknown): projectId is string => + typeof projectId === 'string' && PROJECT_ID.test(projectId) && fs.existsSync(projectDir(projectId)); + const projectName = async (projectId: string) => { + const [first] = await readNotebookFiles(projectDir(projectId)); + + return first ? deserializeDeepnoteFile(first.content.toString('utf8')).project.name : projectId; + }; + + let url = ''; + const staticFiles = new Map(); + const staticFilesOf = (projectId: string): Schemas['StaticFilesSettings'] => + staticFiles.get(projectId) ?? { + sharingEnabled: false, + apiAccessEnabled: false, + published: false, + url: `${url}/apps/${projectId}/` + }; + const app = express(); app.use((req, res, next) => @@ -102,9 +168,7 @@ export async function startMockDeepnoteCloud(rootDir?: string): Promise - PROJECT_ID.test(projectId) && fs.existsSync(projectDir(projectId)) - ? next() - : sendError(res, 404, 'Project not found') + isKnownProject(projectId) ? next() : sendError(res, 404, 'Project not found') ); app.get('/v2/me', (_req, res) => { @@ -136,12 +200,11 @@ export async function startMockDeepnoteCloud(rootDir?: string): Promise name); const projects = await Promise.all( ids.map(async (id): Promise => { - const [first] = await readNotebookFiles(projectDir(id)); const { birthtime, mtime } = await fs.promises.stat(projectDir(id)); return { id, - name: first ? deserializeDeepnoteFile(first.content.toString('utf8')).project.name : id, + name: await projectName(id), projectType: 'standard', createdAt: birthtime.toISOString(), updatedAt: mtime.toISOString(), @@ -159,6 +222,126 @@ export async function startMockDeepnoteCloud(rootDir?: string): Promise { + const { projectId } = req.params; + const { birthtime, mtime } = await fs.promises.stat(projectDir(projectId)); + const body: Schemas['GetProjectResponse'] = { + project: { + id: projectId, + name: await projectName(projectId), + projectType: 'standard', + createdAt: birthtime.toISOString(), + updatedAt: mtime.toISOString(), + folder: null, + // publishApp reads only the files and the static-file settings. + notebooks: [], + integrations: [], + files: await listProjectFiles(filesDir(projectId)), + staticFiles: staticFilesOf(projectId), + streamlitAppApiAccessEnabled: false, + environment: null + } + }; + + res.json(body); + }); + + app.patch('/v2/projects/:projectId', express.json(), async (req, res) => { + const { projectId } = req.params; + const update = (req.body as UpdateProjectBody | undefined)?.staticFiles; + if (update?.sharingEnabled === undefined && update?.apiAccessEnabled === undefined) { + return sendError(res, 400, 'At least one setting is required'); + } + + const current = staticFilesOf(projectId); + const sharingEnabled = update.sharingEnabled ?? current.sharingEnabled; + if (update.apiAccessEnabled === true && !sharingEnabled) { + return sendError(res, 400, 'API access can be enabled only when sharing is enabled'); + } + + const settings: Schemas['StaticFilesSettings'] = { + ...current, + sharingEnabled, + apiAccessEnabled: sharingEnabled && (update.apiAccessEnabled ?? current.apiAccessEnabled) + }; + staticFiles.set(projectId, settings); + + const { birthtime, mtime } = await fs.promises.stat(projectDir(projectId)); + const body: Schemas['UpdateProjectResponse'] = { + project: { + id: projectId, + name: await projectName(projectId), + projectType: 'standard', + createdAt: birthtime.toISOString(), + updatedAt: mtime.toISOString(), + staticFiles: settings, + streamlitAppApiAccessEnabled: false, + environment: null, + folder: null + } + }; + + res.json(body); + }); + + app.post('/v2/files', express.raw({ type: 'multipart/form-data', limit: MAX_FILE_SIZE }), async (req, res) => { + let form: FormData; + try { + form = await new Request('http://mock.invalid/', { + method: 'POST', + headers: { 'content-type': req.headers['content-type'] ?? '' }, + body: new Uint8Array(req.body as Buffer) + }).formData(); + } catch { + return sendError(res, 400, 'The body is not a readable multipart form'); + } + + const projectId = form.get('projectId'); + const filePath = form.get('path'); + const file = form.get('file'); + if (typeof filePath !== 'string' || !(file instanceof Blob) || typeof projectId !== 'string') { + return sendError(res, 400, 'projectId, path and file are required'); + } + if (!isSafeRelativeFilePath(filePath)) { + return sendError(res, 400, `Unsafe file path: ${filePath}`); + } + if (!isKnownProject(projectId)) { + return sendError(res, 404, 'Project not found'); + } + + const storedPath = freeFilePath(filesDir(projectId), filePath); + const absolute = path.join(filesDir(projectId), ...storedPath.split('/')); + await fs.promises.mkdir(path.dirname(absolute), { recursive: true }); + await fs.promises.writeFile(absolute, Buffer.from(await file.arrayBuffer())); + const { size, mtime } = await fs.promises.stat(absolute); + const body: Schemas['CreateFileResponse'] = { + file: { projectId, path: storedPath, size, updatedAt: mtime.toISOString() } + }; + + res.status(201).json(body); + }); + + app.delete('/v2/files', async (req, res) => { + const { projectId, path: filePath } = req.query; + if (typeof filePath !== 'string' || !isSafeRelativeFilePath(filePath) || typeof projectId !== 'string') { + return sendError(res, 400, 'projectId and a safe path are required'); + } + if (!isKnownProject(projectId)) { + return sendError(res, 404, 'Project not found'); + } + + try { + await fs.promises.unlink(path.join(filesDir(projectId), ...filePath.split('/'))); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return sendError(res, 404, 'File not found'); + } + throw error; + } + + res.status(204).end(); + }); + app.get('/v2/projects/:projectId/export', async (req, res) => { const files = await readNotebookFiles(projectDir(req.params.projectId)); @@ -234,10 +417,11 @@ export async function startMockDeepnoteCloud(rootDir?: string): Promise { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); if (!rootDir) { diff --git a/test/e2e/suite/cloud/mockDeepnoteCloud.e2e.test.ts b/test/e2e/suite/cloud/mockDeepnoteCloud.e2e.test.ts index 83a82d2cf..efa846d47 100644 --- a/test/e2e/suite/cloud/mockDeepnoteCloud.e2e.test.ts +++ b/test/e2e/suite/cloud/mockDeepnoteCloud.e2e.test.ts @@ -152,3 +152,75 @@ describe('Deepnote Cloud — the mock rejects the imports Deepnote Cloud rejects expect(readProjectFiles()).to.deep.equal({ 'main.deepnote': MAIN }); }); }); + +describe('Deepnote Cloud — the mock serves the files and settings that publishing uses', function () { + let cloud: MockDeepnoteCloud; + + async function request(method: string, route: string, body?: FormData | object): Promise { + return fetch(`${cloud.url}${route}`, { + method, + headers: { + Authorization: 'Bearer test-token', + ...(body && !(body instanceof FormData) ? { 'Content-Type': 'application/json' } : {}) + }, + body: body instanceof FormData ? body : body && JSON.stringify(body) + }); + } + + function upload(filePath: string, content: string): Promise { + const form = new FormData(); + form.set('projectId', PROJECT_ID); + form.set('path', filePath); + form.set('file', new Blob([content]), path.posix.basename(filePath)); + + return request('POST', '/v2/files', form); + } + + beforeEach(async function () { + cloud = await startMockDeepnoteCloud(); + const projectDir = path.join(cloud.rootDir, PROJECT_ID); + fs.mkdirSync(projectDir); + fs.writeFileSync(path.join(projectDir, 'main.deepnote'), MAIN); + }); + + afterEach(async function () { + await cloud.stop(); + }); + + it('stores an upload to a taken path at a new path, and lists both', async function () { + expect((await upload('_deepnote_static/app.js', 'one')).status).to.equal(201); + + const second = await upload('_deepnote_static/app.js', 'second'); + + expect(second.status).to.equal(201); + expect(((await second.json()) as { file: { path: string } }).file.path).to.equal('_deepnote_static/app (1).js'); + const { project } = (await (await request('GET', `/v2/projects/${PROJECT_ID}`)).json()) as { + project: { files: { path: string; size: number }[] }; + }; + expect(project.files.map(({ path: filePath, size }) => ({ path: filePath, size }))).to.have.deep.members([ + { path: '_deepnote_static/app.js', size: 3 }, + { path: '_deepnote_static/app (1).js', size: 6 } + ]); + }); + + it('answers 404 for a delete of a missing file', async function () { + const response = await request( + 'DELETE', + `/v2/files?projectId=${PROJECT_ID}&path=${encodeURIComponent('_deepnote_static/missing.js')}` + ); + + expect(response.status).to.equal(404); + expect(await response.json()).to.deep.equal({ message: 'File not found' }); + }); + + it('refuses to enable API access while sharing is disabled', async function () { + const response = await request('PATCH', `/v2/projects/${PROJECT_ID}`, { + staticFiles: { apiAccessEnabled: true } + }); + + expect(response.status).to.equal(400); + expect(await response.json()).to.deep.equal({ + message: 'API access can be enabled only when sharing is enabled' + }); + }); +}); diff --git a/test/e2e/suite/cloud/publishApp.e2e.test.ts b/test/e2e/suite/cloud/publishApp.e2e.test.ts new file mode 100644 index 000000000..329af4c5e --- /dev/null +++ b/test/e2e/suite/cloud/publishApp.e2e.test.ts @@ -0,0 +1,247 @@ +import { expect } from 'chai'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + EditorView, + InputBox, + NotificationType, + SideBarView, + VSBrowser, + Workbench, + type ViewItem +} from 'vscode-extension-tester'; + +import { + API_TOKEN, + QUICK_PICK_TIMEOUT, + SUITE_TIMEOUT, + WORKBENCH_TIMEOUT, + confirmModalDialog, + copyFixtureIntoDir, + openActivityBarView, + openFolderViaDialog, + selectDeepnoteContextMenu, + setApiToken +} from '../../helpers'; +import { startMockDeepnoteCloud, type MockDeepnoteCloud } from '../../helpers/mockDeepnoteCloud'; + +const FIXTURE = 'cloud-sync.deepnote'; +const PROJECT_ID = '4f0c2a8e-6b1d-4c3e-9a7f-2d5e8b1c0a93'; +const PROJECT_NAME = 'E2E Cloud Sync'; +const SYNCED_PROJECT_DIR = 'Sales'; +const CHOOSE_PROJECT = 'Choose the Deepnote project to publish to'; +const CHOOSE_API_ACCESS = 'Should the app be able to call the Deepnote API?'; +const PUBLISH_OUTCOME = /^(App published|Publishing the app failed)/; + +interface StaticFiles { + apiAccessEnabled: boolean; + sharingEnabled: boolean; +} + +/** + * VS Code reuses one quick-pick widget, so the previous quick pick can still be on screen until `placeHolder` shows. + * Clicking an item while the list is being replaced can close the quick pick without accepting it, so the item is + * filtered down to and accepted with Enter instead. + */ +async function pick(placeHolder: string, label: string): Promise { + const driver = VSBrowser.instance.driver; + const input = await InputBox.create(QUICK_PICK_TIMEOUT); + await driver.wait( + async () => (await input.getPlaceHolder().catch(() => '')) === placeHolder, + QUICK_PICK_TIMEOUT, + `the quick pick "${placeHolder}" did not open` + ); + + await input.setText(label); + await driver.wait( + async () => { + const items = await input.getQuickPicks().catch(() => []); + + return items.length === 1 && (await items[0].getLabel().catch(() => '')) === label; + }, + QUICK_PICK_TIMEOUT, + `"${label}" is not the only match in the quick pick "${placeHolder}"` + ); + await input.confirm(); +} + +async function findExplorerItem(folderPath: string): Promise { + await openActivityBarView('Explorer'); + const segments = folderPath.split('/'); + const label = segments[segments.length - 1]; + let found: ViewItem | undefined; + await VSBrowser.instance.driver.wait( + async () => { + for (const section of await new SideBarView().getContent().getSections()) { + if (segments.length > 1) { + await section.openItem(...segments.slice(0, -1)).catch(() => undefined); + } + found = await section.findItem(label, segments.length).catch(() => undefined); + if (found) { + return true; + } + } + + return false; + }, + WORKBENCH_TIMEOUT, + `"${folderPath}" did not appear in the file explorer` + ); + + return found!; +} + +/** + * A toast hides itself after about ten seconds, which one slow poll of the toasts can outlast, so the outcome is read + * from the Notifications Center, which keeps it. + */ +async function waitForPublishOutcome(): Promise { + const center = await new Workbench().openNotificationsCenter(); + let message = ''; + try { + await VSBrowser.instance.driver.wait( + async () => { + for (const notification of await center.getNotifications(NotificationType.Any).catch(() => [])) { + message = await notification.getMessage().catch(() => ''); + if (PUBLISH_OUTCOME.test(message)) { + return true; + } + } + + return false; + }, + WORKBENCH_TIMEOUT, + `no notification matching ${PUBLISH_OUTCOME}` + ); + } finally { + await center.close().catch(() => undefined); + } + + return message; +} + +async function publishFromExplorer(folderPath: string, answerPrompts: () => Promise): Promise { + // Every publish of a folder ends with the same message, so one left from the previous publish must not count. + await (await new Workbench().openNotificationsCenter()).clearAllNotifications(); + await selectDeepnoteContextMenu(await findExplorerItem(folderPath), 'Publish App…'); + await answerPrompts(); + + const message = await waitForPublishOutcome(); + if (!message.startsWith('App published')) { + throw new Error(message); + } + + return message; +} + +describe('Deepnote Cloud — publishing an app to the mock Deepnote Cloud', function () { + this.timeout(SUITE_TIMEOUT); + + let cloud: MockDeepnoteCloud | undefined; + let workspaceDir = ''; + let cloudAppFile = ''; + const outcomes: string[] = []; + const cloudContent: string[] = []; + let staticFilesAfterFirstPublish: StaticFiles | undefined; + + async function readStaticFiles(): Promise { + const response = await fetch(`${cloud!.url}/v2/projects/${PROJECT_ID}`, { + headers: { Authorization: `Bearer ${API_TOKEN}` } + }); + + return ((await response.json()) as { project: { staticFiles: StaticFiles } }).project.staticFiles; + } + + before(async function () { + cloud = await startMockDeepnoteCloud(); + const cloudProjectDir = path.join(cloud.rootDir, PROJECT_ID); + fs.mkdirSync(cloudProjectDir); + copyFixtureIntoDir(cloudProjectDir, FIXTURE); + cloudAppFile = path.join(cloudProjectDir, 'files', '_deepnote_static', 'index.html'); + + workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-e2e-publish-')); + fs.mkdirSync(path.join(workspaceDir, '.vscode')); + fs.writeFileSync( + path.join(workspaceDir, '.vscode', 'settings.json'), + // Compact folders would merge `Sales/app` into one row, so a right-click could target `Sales`. + JSON.stringify({ 'deepnote.apiUrl': cloud.url, 'explorer.compactFolders': false }) + ); + const siteFile = path.join(workspaceDir, 'site', 'index.html'); + fs.mkdirSync(path.dirname(siteFile)); + fs.writeFileSync(siteFile, '

site v1

'); + const syncedAppFile = path.join(workspaceDir, 'synced', SYNCED_PROJECT_DIR, 'app', 'index.html'); + fs.mkdirSync(path.dirname(syncedAppFile), { recursive: true }); + fs.writeFileSync(syncedAppFile, '

synced app

'); + fs.writeFileSync( + path.join(workspaceDir, 'synced', '.deepnote-sync.json'), + JSON.stringify({ + version: 1, + projects: { [PROJECT_ID]: { dir: SYNCED_PROJECT_DIR, notebooks: [], contentHash: 'x' } } + }) + ); + + await VSBrowser.instance.waitForWorkbench(WORKBENCH_TIMEOUT); + await openFolderViaDialog(workspaceDir); + await VSBrowser.instance.waitForWorkbench(WORKBENCH_TIMEOUT); + await setApiToken(); + + outcomes.push( + await publishFromExplorer('site', async () => { + await pick(CHOOSE_PROJECT, PROJECT_NAME); + await pick(CHOOSE_API_ACCESS, 'API access on'); + }) + ); + cloudContent.push(fs.readFileSync(cloudAppFile, 'utf8')); + staticFilesAfterFirstPublish = await readStaticFiles(); + + fs.writeFileSync(siteFile, '

site v2

'); + outcomes.push( + await publishFromExplorer('site', () => + confirmModalDialog('Publish', { messageIncludes: `Publish site to ${PROJECT_NAME}?` }) + ) + ); + cloudContent.push(fs.readFileSync(cloudAppFile, 'utf8')); + + const syncedApp = `synced/${SYNCED_PROJECT_DIR}/app`; + outcomes.push(await publishFromExplorer(syncedApp, () => pick(CHOOSE_API_ACCESS, 'API access off'))); + // The publish mirrored the app into the synced project's `.files`. Once the explorer shows it, its rows + // stop moving, so the next right-click lands on the app folder. + await findExplorerItem(`synced/${SYNCED_PROJECT_DIR}/.files`); + fs.writeFileSync(cloudAppFile, '

edited in Deepnote

'); + outcomes.push( + await publishFromExplorer(syncedApp, async () => { + await confirmModalDialog('Publish', { messageIncludes: `Publish app to ${SYNCED_PROJECT_DIR}?` }); + await confirmModalDialog('Force', { messageIncludes: 'changed in Deepnote since the last sync' }); + }) + ); + cloudContent.push(fs.readFileSync(cloudAppFile, 'utf8')); + }); + + after(async function () { + await new EditorView().closeAllEditors().catch(() => undefined); + await cloud?.stop(); + if (workspaceDir) { + fs.rmSync(workspaceDir, { recursive: true, force: true }); + } + }); + + it('publishes a folder to the picked project with API access, and shows the app URL', function () { + expect(outcomes[0]).to.equal(`App published: ${cloud!.url}/apps/${PROJECT_ID}/`); + expect(cloudContent[0]).to.equal('

site v1

'); + expect(staticFilesAfterFirstPublish).to.include({ sharingEnabled: true, apiAccessEnabled: true }); + }); + + it('republishes with the saved settings after one confirmation', function () { + expect(outcomes[1]).to.equal(`App published: ${cloud!.url}/apps/${PROJECT_ID}/`); + expect(cloudContent[1]).to.equal('

site v2

'); + }); + + it('overwrites app files changed in Deepnote when the user picks Force', function () { + expect(outcomes.slice(2)).to.deep.equal([ + `App published: ${cloud!.url}/apps/${PROJECT_ID}/`, + `App published: ${cloud!.url}/apps/${PROJECT_ID}/` + ]); + expect(cloudContent[2]).to.equal('

synced app

'); + }); +}); diff --git a/test/e2e/suite/cloud/syncWithDeepnoteCloud.e2e.test.ts b/test/e2e/suite/cloud/syncWithDeepnoteCloud.e2e.test.ts index 9c997fae5..cf98f7b50 100644 --- a/test/e2e/suite/cloud/syncWithDeepnoteCloud.e2e.test.ts +++ b/test/e2e/suite/cloud/syncWithDeepnoteCloud.e2e.test.ts @@ -13,11 +13,11 @@ import { copyFixtureIntoDir, dismissAllNotifications, openFolderViaDialog, + setApiToken, waitForNotification } from '../../helpers'; import { MOCK_WORKSPACE_NAME, startMockDeepnoteCloud, type MockDeepnoteCloud } from '../../helpers/mockDeepnoteCloud'; -const API_TOKEN = 'e2e_mock_token'; const FIXTURE = 'cloud-sync.deepnote'; const PROJECT_ID = '4f0c2a8e-6b1d-4c3e-9a7f-2d5e8b1c0a93'; const PROJECT_NAME = 'E2E Cloud Sync'; @@ -44,26 +44,6 @@ async function sync(answerPrompts: () => Promise): Promise { return message; } -async function setApiToken(): Promise { - await dismissAllNotifications(); - await new Workbench().executeCommand('Deepnote: Set API Token'); - const input = await InputBox.create(QUICK_PICK_TIMEOUT); - await input.setText(API_TOKEN); - await input.confirm(); - - const outcome = await waitForNotification( - /^(Signed in to the Deepnote workspace|Deepnote API token saved|Deepnote rejected|Could not check|Could not save)/, - WORKBENCH_TIMEOUT, - true - ); - const message = (await outcome?.getMessage()) ?? ''; - if (/^(Deepnote rejected|Could not)/.test(message)) { - throw new Error(message); - } - - return message; -} - function replaceInFile(file: string, from: string, to: string): void { const content = fs.readFileSync(file, 'utf8'); if (!content.includes(from)) {