diff --git a/README.md b/README.md index 075365a..c8999a2 100644 --- a/README.md +++ b/README.md @@ -1348,6 +1348,28 @@ cliente en vez de comprobar nombres de rol. Quien no tenga `pages.publish` solo ve páginas publicadas en el listado. +### Lotes e importación + +``` +POST /api/v1/batch hasta 50 operaciones {method, path, body}; atomic +POST /api/v1/import/{tipo} filas JSON (rows) o CSV (file); match, atomic, dry_run +``` + +Cada operación de un lote pasa por el kernel HTTP como una petición propia con +las credenciales de quien llama: se valida, se autoriza, cuenta para el límite +de peticiones y responde igual que si llegara sola, así que un lote no puede +hacer nada que sus operaciones no pudieran. [`BatchRunner`](app/Support/Batch/BatchRunner.php) +no sabe qué es una página, y es justo lo que lo hace seguro. Con `atomic` va en +una transacción y se para en el primer fallo deshaciéndolo todo; lo que ya salió +de la base (un trabajo en cola, un aviso) no se deshace. + +La importación convierte filas en esas mismas llamadas: con `id`, o con el +campo de `match` (`slug`, `name`, `full_name`) apuntando a una ficha que existe, +actualiza; si no, crea. En CSV, `title.es` rellena el título en castellano, una +lista se escribe `1|2|3`, `true`/`false`/`null` son lo que dicen y una celda +vacía se omite en vez de borrar lo que hubiera. Por defecto es todo o nada, y +`dry_run` lo comprueba todo sin guardar nada. + ### Búsqueda `GET /api/public/v1/search?q=…` busca en lo que la web enseña —páginas, diff --git a/app/Http/Controllers/Api/V1/BatchController.php b/app/Http/Controllers/Api/V1/BatchController.php new file mode 100644 index 0000000..4cbebf7 --- /dev/null +++ b/app/Http/Controllers/Api/V1/BatchController.php @@ -0,0 +1,99 @@ +validate([ + 'operations' => ['required', 'array', 'min:1', 'max:'.BatchRunner::MAX_OPERATIONS], + 'operations.*.method' => ['required', Rule::in(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'get', 'post', 'put', 'patch', 'delete'])], + 'operations.*.path' => [ + 'required', + 'string', + 'max:500', + function (string $attribute, mixed $value, Closure $fail): void { + if (! is_string($value) || ! BatchRunner::allows($value)) { + $fail(__('rules.batch.path')); + } + }, + ], + 'operations.*.body' => ['sometimes', 'nullable', 'array'], + 'atomic' => ['sometimes', 'boolean'], + ]); + + $outcome = $runner->run($request, $data['operations'], atomic: $request->boolean('atomic')); + + return $this->answer($outcome); + } + + /** + * Rows as JSON (`rows`) or a CSV file (`file`). `match` names the field + * that finds an existing record to update; `dry_run` validates everything + * and keeps nothing. + */ + public function import(Request $request, string $resource, BatchRunner $runner, Importer $importer): JsonResponse + { + abort_unless(array_key_exists($resource, AdminContent::WITH_HISTORY), 404); + + $request->validate([ + 'rows' => ['required_without:file', 'array'], + 'rows.*' => ['array'], + 'file' => ['required_without:rows', 'file', 'mimes:csv,txt', 'max:2048'], + 'match' => ['sometimes', 'nullable', 'string', Rule::in(['slug', 'name', 'full_name'])], + 'atomic' => ['sometimes', 'boolean'], + 'dry_run' => ['sometimes', 'boolean'], + ]); + + $rows = $request->hasFile('file') + ? $importer->rowsFromCsv($request->file('file')) + : $request->input('rows'); + + $operations = $importer->operations($resource, $rows, $request->input('match')); + + // A dry run is all-or-nothing by nature: it keeps nothing either way. + $outcome = $runner->run( + $request, + $operations, + atomic: $request->boolean('atomic', true) || $request->boolean('dry_run'), + rollback: $request->boolean('dry_run'), + ); + + return $this->answer($outcome, ['dry_run' => $request->boolean('dry_run')]); + } + + /** + * @param array{results: array>, committed: bool} $outcome + * @param array $meta + */ + protected function answer(array $outcome, array $meta = []): JsonResponse + { + $failed = collect($outcome['results'])->where('status', '>=', 400)->count(); + + return response()->json([ + 'data' => $outcome['results'], + 'meta' => [ + ...$meta, + 'committed' => $outcome['committed'], + 'succeeded' => count($outcome['results']) - $failed, + 'failed' => $failed, + ], + ]); + } +} diff --git a/app/Support/Batch/BatchAborted.php b/app/Support/Batch/BatchAborted.php new file mode 100644 index 0000000..5609b50 --- /dev/null +++ b/app/Support/Batch/BatchAborted.php @@ -0,0 +1,12 @@ +|null}> $operations + * @param bool $atomic All or nothing. + * @param bool $rollback Undo everything even when all succeeded — a dry run. + * @return array{results: array>, committed: bool} + */ + public function run(Request $parent, array $operations, bool $atomic = false, bool $rollback = false): array + { + $results = []; + + $execute = function () use ($parent, $operations, $atomic, &$results): void { + foreach ($operations as $index => $operation) { + $result = $this->dispatch($parent, $operation); + $results[] = ['index' => $index, ...$result]; + + if ($atomic && $result['status'] >= 400) { + throw new BatchAborted; + } + } + }; + + if (! $atomic && ! $rollback) { + $execute(); + + return ['results' => $results, 'committed' => true]; + } + + try { + DB::transaction(function () use ($execute, $rollback): void { + $execute(); + + if ($rollback) { + throw new BatchAborted; + } + }); + } catch (BatchAborted) { + return ['results' => $results, 'committed' => false]; + } + + return ['results' => $results, 'committed' => true]; + } + + /** + * @param array{method: string, path: string, body?: array|null} $operation + * @return array{method: string, path: string, status: int, body: mixed} + */ + protected function dispatch(Request $parent, array $operation): array + { + $method = strtoupper($operation['method']); + $path = $operation['path']; + + $request = Request::create( + '/api'.$path, + $method, + server: array_filter([ + 'HTTP_AUTHORIZATION' => $parent->header('Authorization'), + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_ACCEPT_LANGUAGE' => $parent->header('Accept-Language'), + 'CONTENT_TYPE' => 'application/json', + 'REMOTE_ADDR' => $parent->ip(), + ]), + content: $method === 'GET' ? null : (string) json_encode((object) ($operation['body'] ?? [])), + ); + + try { + $response = $this->kernel->handle($request); + } catch (Throwable $e) { + report($e); + + return ['method' => $method, 'path' => $path, 'status' => 500, 'body' => null]; + } finally { + // The kernel swapped the current request for the operation's; + // whatever runs after this one still belongs to the batch's. + app()->instance('request', $parent); + } + + $content = $response->getContent(); + + return [ + 'method' => $method, + 'path' => $path, + 'status' => $response->getStatusCode(), + 'body' => $content === '' || $content === false ? null : json_decode($content, true), + ]; + } +} diff --git a/app/Support/Batch/Importer.php b/app/Support/Batch/Importer.php new file mode 100644 index 0000000..f81f4b6 --- /dev/null +++ b/app/Support/Batch/Importer.php @@ -0,0 +1,148 @@ +> + */ + public function rowsFromCsv(UploadedFile $file): array + { + $handle = fopen($file->getRealPath(), 'r'); + $header = null; + $rows = []; + + while (($line = fgetcsv($handle, escape: '\\')) !== false) { + if ($line === [null]) { + continue; + } + + if ($header === null) { + // A byte-order mark is what a spreadsheet saving "CSV UTF-8" + // puts before the first column's name. + $header = array_map(fn ($name) => trim((string) preg_replace('/^\xEF\xBB\xBF/', '', (string) $name)), $line); + + continue; + } + + $row = []; + + foreach ($header as $i => $column) { + $value = $line[$i] ?? ''; + + if ($column === '' || $value === '') { + continue; + } + + Arr::set($row, $column, $this->cell($column, $value)); + } + + $rows[] = $row; + } + + fclose($handle); + + return $rows; + } + + /** + * @param array> $rows + * @return array}> + */ + public function operations(string $resource, array $rows, ?string $match = null): array + { + if (count($rows) > self::MAX_ROWS) { + throw ValidationException::withMessages(['rows' => [__('rules.batch.too_many', ['max' => self::MAX_ROWS])]]); + } + + return array_map(function (array $row) use ($resource, $match): array { + $id = $row['id'] ?? $this->existing($resource, $row, $match); + unset($row['id']); + + return $id === null + ? ['method' => 'POST', 'path' => "/v1/{$resource}", 'body' => $row] + : ['method' => 'PATCH', 'path' => "/v1/{$resource}/{$id}", 'body' => $row]; + }, $rows); + } + + /** + * The record a row names through its `match` field, if any: a slug is + * looked up in the default language when it is translated. + * + * @param array $row + */ + protected function existing(string $resource, array $row, ?string $match): mixed + { + if ($match === null || ! array_key_exists($match, $row)) { + return null; + } + + $model = AdminContent::model($resource); + $instance = new $model; + $value = $row[$match]; + $translated = in_array(HasTranslations::class, class_uses_recursive($model), true) + && in_array($match, $instance->translatable, true); + + if ($translated) { + $locale = (string) config('locales.default'); + $value = is_array($value) ? ($value[$locale] ?? null) : $value; + $column = "{$match}->{$locale}"; + } else { + $column = $match; + } + + if ($value === null || is_array($value)) { + return null; + } + + $query = $model::query(); + + if (method_exists($instance, 'bootSoftDeletes')) { + $query->withTrashed(); + } + + return $query->where($column, $value)->value($instance->getKeyName()); + } + + protected function cell(string $column, string $value): mixed + { + $base = explode('.', $column)[0]; + + return match (true) { + in_array($base, self::LISTS, true) => array_map( + fn ($item) => is_numeric($item) ? (int) $item : $item, + array_values(array_filter(array_map('trim', explode('|', $value)), fn ($item) => $item !== '')), + ), + strtolower($value) === 'true' => true, + strtolower($value) === 'false' => false, + strtolower($value) === 'null' => null, + default => $value, + }; + } +} diff --git a/lang/ca/rules.php b/lang/ca/rules.php index fbf0686..a8d6774 100644 --- a/lang/ca/rules.php +++ b/lang/ca/rules.php @@ -39,6 +39,11 @@ return [ 'not_awaiting' => 'No està esperant revisió.', ], + 'batch' => [ + 'path' => 'Només operacions de l\'API d\'administració (/v1/…), sense lots niats ni importacions.', + 'too_many' => 'Com a molt :max files per importació.', + ], + 'settings' => [ 'read_only' => 'Aquest ajust l\'escriu l\'aplicació i no es pot editar a mà.', ], diff --git a/lang/en/rules.php b/lang/en/rules.php index 9c630fc..bb0053d 100644 --- a/lang/en/rules.php +++ b/lang/en/rules.php @@ -39,6 +39,11 @@ return [ 'not_awaiting' => 'It is not awaiting review.', ], + 'batch' => [ + 'path' => 'Only admin API operations (/v1/…), with no nested batches or imports.', + 'too_many' => 'At most :max rows per import.', + ], + 'settings' => [ 'read_only' => 'This setting is written by the application and cannot be edited by hand.', ], diff --git a/lang/es/rules.php b/lang/es/rules.php index 60adb83..41f37ce 100644 --- a/lang/es/rules.php +++ b/lang/es/rules.php @@ -39,6 +39,11 @@ return [ 'not_awaiting' => 'No está esperando revisión.', ], + 'batch' => [ + 'path' => 'Solo operaciones de la API de administración (/v1/…), sin lotes anidados ni importaciones.', + 'too_many' => 'Como mucho :max filas por importación.', + ], + 'settings' => [ 'read_only' => 'Este ajuste lo escribe la aplicación y no se puede editar a mano.', ], diff --git a/public/openapi.yaml b/public/openapi.yaml index 2f4baad..4eabf60 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -136,6 +136,8 @@ tags: description: Review, publication and preview links - name: Health description: What is wrong with the site, and what is left to translate + - name: Batch + description: Several operations in one, and importing records from JSON or CSV - name: Users description: Local user administration - name: Media @@ -3638,6 +3640,107 @@ paths: items: { $ref: "#/components/schemas/PendingTranslation" } "422": { $ref: "#/components/responses/ValidationError" } + /v1/batch: + post: + tags: [Batch] + summary: Run several operations + description: | + Up to 50 admin API calls in one request. Each runs as a request of its + own with the caller's credentials — validated, authorised and answered + exactly as if sent separately — so a batch can do nothing its + operations would be refused. Their answers come back in order. + + With `atomic`, it stops at the first failure and undoes everything + before it. What already left the database — a queued job, a + notification — is not undone. + + `path` is an admin path such as `/v1/pages/3`; batches do not nest and + do not reach the import or the token exchange. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [operations] + properties: + atomic: { type: boolean, default: false } + operations: + type: array + maxItems: 50 + items: + type: object + required: [method, path] + properties: + method: { type: string, enum: [GET, POST, PUT, PATCH, DELETE] } + path: { type: string, example: /v1/technologies } + body: { type: object, nullable: true, additionalProperties: true } + responses: + "200": + description: Each operation's answer + content: + application/json: + schema: { $ref: "#/components/schemas/BatchOutcome" } + "403": { $ref: "#/components/responses/Forbidden" } + "422": { $ref: "#/components/responses/ValidationError" } + + /v1/import/{resource}: + post: + tags: [Batch] + summary: Import records + description: | + Rows of one kind of record, as JSON (`rows`) or a CSV `file`, turned + into the create and update calls the API already has — up to 50 per + import, each validated and authorised like any other call. + + A row with an `id`, or whose `match` field (`slug`, `name` or + `full_name`) names an existing record, updates it; any other row + creates one. In a CSV, a column `title.es` fills the Spanish title, a + list is written `1|2|3`, `true`/`false`/`null` are what they say, and + an empty cell is left out rather than erasing what is there. + + All-or-nothing unless `atomic` is false. `dry_run` checks every row and + keeps none. + parameters: + - name: resource + in: path + required: true + schema: + type: string + enum: [pages, categories, projects, members, services, clients, testimonials, talks, technologies, calls] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [rows] + properties: + rows: + type: array + maxItems: 50 + items: { type: object, additionalProperties: true, description: The same body the resource's create endpoint takes. } + match: { type: string, enum: [slug, name, full_name], nullable: true } + atomic: { type: boolean, default: true } + dry_run: { type: boolean, default: false } + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: { type: string, format: binary, description: "CSV with a header row, UTF-8." } + match: { type: string, enum: [slug, name, full_name] } + atomic: { type: boolean, default: true } + dry_run: { type: boolean, default: false } + responses: + "200": + description: Each row's answer + content: + application/json: + schema: { $ref: "#/components/schemas/BatchOutcome" } + "403": { $ref: "#/components/responses/Forbidden" } + "422": { $ref: "#/components/responses/ValidationError" } + /v1/media: get: tags: [Media] @@ -5418,6 +5521,27 @@ components: missing: { type: array, items: { type: string }, description: Languages with nothing written. } stale: { type: object, additionalProperties: { type: string, format: date-time }, description: Language → since when it has been behind. } + BatchOutcome: + type: object + properties: + data: + type: array + items: + type: object + properties: + index: { type: integer } + method: { type: string } + path: { type: string } + status: { type: integer, description: The HTTP status the operation answered with. } + body: { nullable: true, description: What the operation answered. } + meta: + type: object + properties: + committed: { type: boolean, description: False when an atomic run or a dry run undid everything. } + succeeded: { type: integer } + failed: { type: integer } + dry_run: { type: boolean, nullable: true } + MediaFor: type: string enum: [pages, technologies, projects, members, clients, testimonials] diff --git a/routes/api.php b/routes/api.php index bec7a26..21fd0c7 100644 --- a/routes/api.php +++ b/routes/api.php @@ -12,6 +12,7 @@ use App\Http\Controllers\Api\PublicApi\V1\ShowcaseController; use App\Http\Controllers\Api\PublicApi\V1\SiteController; use App\Http\Controllers\Api\PublicApi\V1\TelemetryController; use App\Http\Controllers\Api\V1\ApplicationController as AdminApplicationController; +use App\Http\Controllers\Api\V1\BatchController; use App\Http\Controllers\Api\V1\CallController; use App\Http\Controllers\Api\V1\CategoryController; use App\Http\Controllers\Api\V1\ClientController; @@ -246,6 +247,13 @@ Route::prefix('v1')->name('api.v1.')->group(function () { Route::get('reviews', [ReviewController::class, 'queue'])->name('reviews.queue'); + // Several calls in one. Each operation is still its own request, with + // its own validation and permission check. + Route::post('batch', [BatchController::class, 'batch'])->name('batch'); + Route::post('import/{resource}', [BatchController::class, 'import']) + ->whereIn('resource', array_keys(AdminContent::WITH_HISTORY)) + ->name('import'); + // The panel's health screen, and what is left to translate. Route::get('health', [HealthController::class, 'index'])->name('health'); Route::get('translations/pending', [HealthController::class, 'translations'])->name('translations.pending'); diff --git a/tests/Feature/Api/BatchApiTest.php b/tests/Feature/Api/BatchApiTest.php new file mode 100644 index 0000000..b311e32 --- /dev/null +++ b/tests/Feature/Api/BatchApiTest.php @@ -0,0 +1,181 @@ +seed(RolePermissionSeeder::class); + } + + protected function actingWithRole(string $role): User + { + $user = User::factory()->create(); + $user->syncRoles([$role]); + + $this->withHeaders($this->apiHeadersFor($user)); + + return $user; + } + + #[Test] + public function a_batch_runs_each_operation_and_reports_each_answer(): void + { + $this->actingWithRole('editor'); + + $member = Member::factory()->create(); + + $this->postJson('/api/v1/batch', [ + 'operations' => [ + ['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => 'Vue', 'slug' => 'vue']], + ['method' => 'PATCH', 'path' => "/v1/members/{$member->id}", 'body' => ['github_username' => 'ada']], + ['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => '']], + ['method' => 'GET', 'path' => '/v1/me'], + ], + ]) + ->assertOk() + ->assertJsonPath('data.0.status', 201) + ->assertJsonPath('data.0.body.data.slug', 'vue') + ->assertJsonPath('data.1.status', 200) + ->assertJsonPath('data.2.status', 422) + ->assertJsonPath('data.3.status', 200) + ->assertJsonPath('meta.committed', true) + ->assertJsonPath('meta.failed', 1); + + $this->assertSame('ada', $member->fresh()->github_username); + $this->assertTrue(Technology::query()->where('slug', 'vue')->exists()); + } + + #[Test] + public function an_atomic_batch_keeps_nothing_when_one_fails(): void + { + $this->actingWithRole('editor'); + + $this->postJson('/api/v1/batch', [ + 'atomic' => true, + 'operations' => [ + ['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => 'Vue', 'slug' => 'vue']], + ['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => '']], + ['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => 'Never', 'slug' => 'never']], + ], + ]) + ->assertOk() + ->assertJsonCount(2, 'data') + ->assertJsonPath('meta.committed', false); + + $this->assertSame(0, Technology::query()->count()); + } + + #[Test] + public function a_batch_cannot_do_what_its_caller_cannot(): void + { + $this->actingWithRole('viewer'); + + $this->postJson('/api/v1/batch', [ + 'operations' => [['method' => 'POST', 'path' => '/v1/technologies', 'body' => ['name' => 'Vue', 'slug' => 'vue']]], + ])->assertJsonPath('data.0.status', 403); + + $this->assertSame(0, Technology::query()->count()); + } + + #[Test] + public function batches_do_not_nest_or_leave_the_admin_api(): void + { + $this->actingWithRole('editor'); + + foreach (['/v1/batch', '/v1/import/pages', '/v1/auth/token', '/public/v1/site', '/v1/../public/v1/site'] as $path) { + $this->postJson('/api/v1/batch', ['operations' => [['method' => 'POST', 'path' => $path]]]) + ->assertUnprocessable() + ->assertJsonValidationErrors('operations.0.path'); + } + } + + #[Test] + public function an_import_creates_new_rows_and_updates_matching_ones(): void + { + $this->actingWithRole('editor'); + + Technology::factory()->create(['name' => 'Angular', 'slug' => 'angular', 'website' => null]); + + $this->postJson('/api/v1/import/technologies', [ + 'match' => 'slug', + 'rows' => [ + ['name' => 'Angular', 'slug' => 'angular', 'website' => 'https://angular.dev'], + ['name' => 'Svelte', 'slug' => 'svelte', 'description' => ['es' => 'Compilador', 'en' => 'Compiler']], + ], + ]) + ->assertOk() + ->assertJsonPath('data.0.method', 'PATCH') + ->assertJsonPath('data.1.method', 'POST') + ->assertJsonPath('meta.succeeded', 2); + + $this->assertSame('https://angular.dev', Technology::query()->where('slug', 'angular')->value('website')); + $this->assertSame(2, Technology::query()->count()); + } + + #[Test] + public function a_csv_import_reads_languages_lists_and_booleans(): void + { + $this->actingWithRole('editor'); + + [$vue, $php] = Technology::factory()->count(2)->create(); + + $csv = "\xEF\xBB\xBFname,slug,headline.es,headline.en,technology_ids,is_public,github_username\n" + ."Ada Lovelace,ada,Backend,Backend dev,{$vue->id}|{$php->id},true,\n" + ."Linus,linus,Sistemas,,,false,linus\n"; + + $this->post('/api/v1/import/members', [ + 'file' => UploadedFile::fake()->createWithContent('members.csv', $csv), + ], ['Accept' => 'application/json']) + ->assertOk() + ->assertJsonPath('meta.failed', 0); + + $ada = Member::query()->where('slug', 'ada')->sole(); + + $this->assertSame('Backend dev', $ada->getTranslation('headline', 'en')); + $this->assertCount(2, $ada->technologies); + $this->assertTrue($ada->is_public); + $this->assertFalse(Member::query()->where('slug', 'linus')->sole()->is_public); + } + + #[Test] + public function a_dry_run_validates_everything_and_keeps_nothing(): void + { + $this->actingWithRole('editor'); + + $this->postJson('/api/v1/import/technologies', [ + 'dry_run' => true, + 'rows' => [ + ['name' => 'Vue', 'slug' => 'vue'], + ['name' => 'Svelte', 'slug' => 'svelte'], + ], + ]) + ->assertOk() + ->assertJsonPath('meta.dry_run', true) + ->assertJsonPath('meta.committed', false) + ->assertJsonPath('meta.succeeded', 2); + + $this->assertSame(0, Technology::query()->count()); + } +}