diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8a8f0e5c..9d89063f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -504,6 +504,7 @@ jobs: run: | echo "ERROR: Generated migration files not up to date!" echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + cat ./src/migrations/*-TestMigration.ts exit 1 - name: Run SQL generation diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index d006ef38b..e86ac9335 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -201,8 +201,12 @@ Class | Method | HTTP request | Description *StacksApi* | [**getStack**](doc//StacksApi.md#getstack) | **GET** /stacks/{id} | *StacksApi* | [**searchStacks**](doc//StacksApi.md#searchstacks) | **GET** /stacks | *StacksApi* | [**updateStack**](doc//StacksApi.md#updatestack) | **PUT** /stacks/{id} | +*SyncApi* | [**deleteSyncAck**](doc//SyncApi.md#deletesyncack) | **DELETE** /sync/ack | *SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **POST** /sync/delta-sync | *SyncApi* | [**getFullSyncForUser**](doc//SyncApi.md#getfullsyncforuser) | **POST** /sync/full-sync | +*SyncApi* | [**getSyncAck**](doc//SyncApi.md#getsyncack) | **GET** /sync/ack | +*SyncApi* | [**getSyncStream**](doc//SyncApi.md#getsyncstream) | **POST** /sync/stream | +*SyncApi* | [**sendSyncAck**](doc//SyncApi.md#sendsyncack) | **POST** /sync/ack | *SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config | *SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults | *SystemConfigApi* | [**getStorageTemplateOptions**](doc//SystemConfigApi.md#getstoragetemplateoptions) | **GET** /system-config/storage-template-options | @@ -413,6 +417,14 @@ Class | Method | HTTP request | Description - [StackCreateDto](doc//StackCreateDto.md) - [StackResponseDto](doc//StackResponseDto.md) - [StackUpdateDto](doc//StackUpdateDto.md) + - [SyncAckDeleteDto](doc//SyncAckDeleteDto.md) + - [SyncAckDto](doc//SyncAckDto.md) + - [SyncAckSetDto](doc//SyncAckSetDto.md) + - [SyncEntityType](doc//SyncEntityType.md) + - [SyncRequestType](doc//SyncRequestType.md) + - [SyncStreamDto](doc//SyncStreamDto.md) + - [SyncUserDeleteV1](doc//SyncUserDeleteV1.md) + - [SyncUserV1](doc//SyncUserV1.md) - [SystemConfigBackupsDto](doc//SystemConfigBackupsDto.md) - [SystemConfigDto](doc//SystemConfigDto.md) - [SystemConfigFFmpegDto](doc//SystemConfigFFmpegDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 2a2b6d46a..e5794a269 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -226,6 +226,14 @@ part 'model/source_type.dart'; part 'model/stack_create_dto.dart'; part 'model/stack_response_dto.dart'; part 'model/stack_update_dto.dart'; +part 'model/sync_ack_delete_dto.dart'; +part 'model/sync_ack_dto.dart'; +part 'model/sync_ack_set_dto.dart'; +part 'model/sync_entity_type.dart'; +part 'model/sync_request_type.dart'; +part 'model/sync_stream_dto.dart'; +part 'model/sync_user_delete_v1.dart'; +part 'model/sync_user_v1.dart'; part 'model/system_config_backups_dto.dart'; part 'model/system_config_dto.dart'; part 'model/system_config_f_fmpeg_dto.dart'; diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart index f94eb8808..49a4963bf 100644 --- a/mobile/openapi/lib/api/sync_api.dart +++ b/mobile/openapi/lib/api/sync_api.dart @@ -16,6 +16,45 @@ class SyncApi { final ApiClient apiClient; + /// Performs an HTTP 'DELETE /sync/ack' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncAckDeleteDto] syncAckDeleteDto (required): + Future deleteSyncAckWithHttpInfo(SyncAckDeleteDto syncAckDeleteDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody = syncAckDeleteDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'DELETE', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncAckDeleteDto] syncAckDeleteDto (required): + Future deleteSyncAck(SyncAckDeleteDto syncAckDeleteDto,) async { + final response = await deleteSyncAckWithHttpInfo(syncAckDeleteDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + /// Performs an HTTP 'POST /sync/delta-sync' operation and returns the [Response]. /// Parameters: /// @@ -112,4 +151,126 @@ class SyncApi { } return null; } + + /// Performs an HTTP 'GET /sync/ack' operation and returns the [Response]. + Future getSyncAckWithHttpInfo() async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + path, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + Future?> getSyncAck() async { + final response = await getSyncAckWithHttpInfo(); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + final responseBody = await _decodeBodyBytes(response); + return (await apiClient.deserializeAsync(responseBody, 'List') as List) + .cast() + .toList(growable: false); + + } + return null; + } + + /// Performs an HTTP 'POST /sync/stream' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncStreamDto] syncStreamDto (required): + Future getSyncStreamWithHttpInfo(SyncStreamDto syncStreamDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/stream'; + + // ignore: prefer_final_locals + Object? postBody = syncStreamDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncStreamDto] syncStreamDto (required): + Future getSyncStream(SyncStreamDto syncStreamDto,) async { + final response = await getSyncStreamWithHttpInfo(syncStreamDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + + /// Performs an HTTP 'POST /sync/ack' operation and returns the [Response]. + /// Parameters: + /// + /// * [SyncAckSetDto] syncAckSetDto (required): + Future sendSyncAckWithHttpInfo(SyncAckSetDto syncAckSetDto,) async { + // ignore: prefer_const_declarations + final path = r'/sync/ack'; + + // ignore: prefer_final_locals + Object? postBody = syncAckSetDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [SyncAckSetDto] syncAckSetDto (required): + Future sendSyncAck(SyncAckSetDto syncAckSetDto,) async { + final response = await sendSyncAckWithHttpInfo(syncAckSetDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } } diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 49fbe9464..54a8959f6 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -508,6 +508,22 @@ class ApiClient { return StackResponseDto.fromJson(value); case 'StackUpdateDto': return StackUpdateDto.fromJson(value); + case 'SyncAckDeleteDto': + return SyncAckDeleteDto.fromJson(value); + case 'SyncAckDto': + return SyncAckDto.fromJson(value); + case 'SyncAckSetDto': + return SyncAckSetDto.fromJson(value); + case 'SyncEntityType': + return SyncEntityTypeTypeTransformer().decode(value); + case 'SyncRequestType': + return SyncRequestTypeTypeTransformer().decode(value); + case 'SyncStreamDto': + return SyncStreamDto.fromJson(value); + case 'SyncUserDeleteV1': + return SyncUserDeleteV1.fromJson(value); + case 'SyncUserV1': + return SyncUserV1.fromJson(value); case 'SystemConfigBackupsDto': return SystemConfigBackupsDto.fromJson(value); case 'SystemConfigDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 6a917201a..1ebf8314a 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -127,6 +127,12 @@ String parameterToString(dynamic value) { if (value is SourceType) { return SourceTypeTypeTransformer().encode(value).toString(); } + if (value is SyncEntityType) { + return SyncEntityTypeTypeTransformer().encode(value).toString(); + } + if (value is SyncRequestType) { + return SyncRequestTypeTypeTransformer().encode(value).toString(); + } if (value is TimeBucketSize) { return TimeBucketSizeTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/sync_ack_delete_dto.dart b/mobile/openapi/lib/model/sync_ack_delete_dto.dart new file mode 100644 index 000000000..998f812f2 --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_delete_dto.dart @@ -0,0 +1,98 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckDeleteDto { + /// Returns a new [SyncAckDeleteDto] instance. + SyncAckDeleteDto({ + this.types = const [], + }); + + List types; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckDeleteDto && + _deepEquality.equals(other.types, types); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (types.hashCode); + + @override + String toString() => 'SyncAckDeleteDto[types=$types]'; + + Map toJson() { + final json = {}; + json[r'types'] = this.types; + return json; + } + + /// Returns a new [SyncAckDeleteDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckDeleteDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckDeleteDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckDeleteDto( + types: SyncEntityType.listFromJson(json[r'types']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckDeleteDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckDeleteDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckDeleteDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckDeleteDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + }; +} + diff --git a/mobile/openapi/lib/model/sync_ack_dto.dart b/mobile/openapi/lib/model/sync_ack_dto.dart new file mode 100644 index 000000000..c7fafa17d --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_dto.dart @@ -0,0 +1,107 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckDto { + /// Returns a new [SyncAckDto] instance. + SyncAckDto({ + required this.ack, + required this.type, + }); + + String ack; + + SyncEntityType type; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckDto && + other.ack == ack && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (ack.hashCode) + + (type.hashCode); + + @override + String toString() => 'SyncAckDto[ack=$ack, type=$type]'; + + Map toJson() { + final json = {}; + json[r'ack'] = this.ack; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [SyncAckDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckDto( + ack: mapValueOfType(json, r'ack')!, + type: SyncEntityType.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'ack', + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/sync_ack_set_dto.dart b/mobile/openapi/lib/model/sync_ack_set_dto.dart new file mode 100644 index 000000000..0d9eedc38 --- /dev/null +++ b/mobile/openapi/lib/model/sync_ack_set_dto.dart @@ -0,0 +1,101 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncAckSetDto { + /// Returns a new [SyncAckSetDto] instance. + SyncAckSetDto({ + this.acks = const [], + }); + + List acks; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncAckSetDto && + _deepEquality.equals(other.acks, acks); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (acks.hashCode); + + @override + String toString() => 'SyncAckSetDto[acks=$acks]'; + + Map toJson() { + final json = {}; + json[r'acks'] = this.acks; + return json; + } + + /// Returns a new [SyncAckSetDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncAckSetDto? fromJson(dynamic value) { + upgradeDto(value, "SyncAckSetDto"); + if (value is Map) { + final json = value.cast(); + + return SyncAckSetDto( + acks: json[r'acks'] is Iterable + ? (json[r'acks'] as Iterable).cast().toList(growable: false) + : const [], + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncAckSetDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncAckSetDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncAckSetDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncAckSetDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'acks', + }; +} + diff --git a/mobile/openapi/lib/model/sync_entity_type.dart b/mobile/openapi/lib/model/sync_entity_type.dart new file mode 100644 index 000000000..ed82205a3 --- /dev/null +++ b/mobile/openapi/lib/model/sync_entity_type.dart @@ -0,0 +1,85 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SyncEntityType { + /// Instantiate a new enum with the provided [value]. + const SyncEntityType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const userV1 = SyncEntityType._(r'UserV1'); + static const userDeleteV1 = SyncEntityType._(r'UserDeleteV1'); + + /// List of all possible values in this [enum][SyncEntityType]. + static const values = [ + userV1, + userDeleteV1, + ]; + + static SyncEntityType? fromJson(dynamic value) => SyncEntityTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncEntityType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [SyncEntityType] to String, +/// and [decode] dynamic data back to [SyncEntityType]. +class SyncEntityTypeTypeTransformer { + factory SyncEntityTypeTypeTransformer() => _instance ??= const SyncEntityTypeTypeTransformer._(); + + const SyncEntityTypeTypeTransformer._(); + + String encode(SyncEntityType data) => data.value; + + /// Decodes a [dynamic value][data] to a SyncEntityType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + SyncEntityType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'UserV1': return SyncEntityType.userV1; + case r'UserDeleteV1': return SyncEntityType.userDeleteV1; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [SyncEntityTypeTypeTransformer] instance. + static SyncEntityTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/sync_request_type.dart b/mobile/openapi/lib/model/sync_request_type.dart new file mode 100644 index 000000000..d7f1bde54 --- /dev/null +++ b/mobile/openapi/lib/model/sync_request_type.dart @@ -0,0 +1,82 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SyncRequestType { + /// Instantiate a new enum with the provided [value]. + const SyncRequestType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const usersV1 = SyncRequestType._(r'UsersV1'); + + /// List of all possible values in this [enum][SyncRequestType]. + static const values = [ + usersV1, + ]; + + static SyncRequestType? fromJson(dynamic value) => SyncRequestTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncRequestType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [SyncRequestType] to String, +/// and [decode] dynamic data back to [SyncRequestType]. +class SyncRequestTypeTypeTransformer { + factory SyncRequestTypeTypeTransformer() => _instance ??= const SyncRequestTypeTypeTransformer._(); + + const SyncRequestTypeTypeTransformer._(); + + String encode(SyncRequestType data) => data.value; + + /// Decodes a [dynamic value][data] to a SyncRequestType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + SyncRequestType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'UsersV1': return SyncRequestType.usersV1; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [SyncRequestTypeTypeTransformer] instance. + static SyncRequestTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/sync_stream_dto.dart b/mobile/openapi/lib/model/sync_stream_dto.dart new file mode 100644 index 000000000..28fd3dfae --- /dev/null +++ b/mobile/openapi/lib/model/sync_stream_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncStreamDto { + /// Returns a new [SyncStreamDto] instance. + SyncStreamDto({ + this.types = const [], + }); + + List types; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncStreamDto && + _deepEquality.equals(other.types, types); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (types.hashCode); + + @override + String toString() => 'SyncStreamDto[types=$types]'; + + Map toJson() { + final json = {}; + json[r'types'] = this.types; + return json; + } + + /// Returns a new [SyncStreamDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncStreamDto? fromJson(dynamic value) { + upgradeDto(value, "SyncStreamDto"); + if (value is Map) { + final json = value.cast(); + + return SyncStreamDto( + types: SyncRequestType.listFromJson(json[r'types']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncStreamDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncStreamDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncStreamDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncStreamDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'types', + }; +} + diff --git a/mobile/openapi/lib/model/sync_user_delete_v1.dart b/mobile/openapi/lib/model/sync_user_delete_v1.dart new file mode 100644 index 000000000..09411cb79 --- /dev/null +++ b/mobile/openapi/lib/model/sync_user_delete_v1.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncUserDeleteV1 { + /// Returns a new [SyncUserDeleteV1] instance. + SyncUserDeleteV1({ + required this.userId, + }); + + String userId; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncUserDeleteV1 && + other.userId == userId; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (userId.hashCode); + + @override + String toString() => 'SyncUserDeleteV1[userId=$userId]'; + + Map toJson() { + final json = {}; + json[r'userId'] = this.userId; + return json; + } + + /// Returns a new [SyncUserDeleteV1] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncUserDeleteV1? fromJson(dynamic value) { + upgradeDto(value, "SyncUserDeleteV1"); + if (value is Map) { + final json = value.cast(); + + return SyncUserDeleteV1( + userId: mapValueOfType(json, r'userId')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncUserDeleteV1.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncUserDeleteV1.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncUserDeleteV1-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncUserDeleteV1.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'userId', + }; +} + diff --git a/mobile/openapi/lib/model/sync_user_v1.dart b/mobile/openapi/lib/model/sync_user_v1.dart new file mode 100644 index 000000000..b9b41bb72 --- /dev/null +++ b/mobile/openapi/lib/model/sync_user_v1.dart @@ -0,0 +1,127 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SyncUserV1 { + /// Returns a new [SyncUserV1] instance. + SyncUserV1({ + required this.deletedAt, + required this.email, + required this.id, + required this.name, + }); + + DateTime? deletedAt; + + String email; + + String id; + + String name; + + @override + bool operator ==(Object other) => identical(this, other) || other is SyncUserV1 && + other.deletedAt == deletedAt && + other.email == email && + other.id == id && + other.name == name; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (deletedAt == null ? 0 : deletedAt!.hashCode) + + (email.hashCode) + + (id.hashCode) + + (name.hashCode); + + @override + String toString() => 'SyncUserV1[deletedAt=$deletedAt, email=$email, id=$id, name=$name]'; + + Map toJson() { + final json = {}; + if (this.deletedAt != null) { + json[r'deletedAt'] = this.deletedAt!.toUtc().toIso8601String(); + } else { + // json[r'deletedAt'] = null; + } + json[r'email'] = this.email; + json[r'id'] = this.id; + json[r'name'] = this.name; + return json; + } + + /// Returns a new [SyncUserV1] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SyncUserV1? fromJson(dynamic value) { + upgradeDto(value, "SyncUserV1"); + if (value is Map) { + final json = value.cast(); + + return SyncUserV1( + deletedAt: mapDateTime(json, r'deletedAt', r''), + email: mapValueOfType(json, r'email')!, + id: mapValueOfType(json, r'id')!, + name: mapValueOfType(json, r'name')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SyncUserV1.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SyncUserV1.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SyncUserV1-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SyncUserV1.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'deletedAt', + 'email', + 'id', + 'name', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 14245e11b..1d0f06599 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5802,6 +5802,107 @@ ] } }, + "/sync/ack": { + "delete": { + "operationId": "deleteSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckDeleteDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "get": { + "operationId": "getSyncAck", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SyncAckDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "post": { + "operationId": "sendSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckSetDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/sync/delta-sync": { "post": { "operationId": "getDeltaSync", @@ -5889,6 +5990,41 @@ ] } }, + "/sync/stream": { + "post": { + "operationId": "getSyncStream", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncStreamDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/system-config": { "get": { "operationId": "getConfig", @@ -11696,6 +11832,113 @@ }, "type": "object" }, + "SyncAckDeleteDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncEntityType" + }, + "type": "array" + } + }, + "type": "object" + }, + "SyncAckDto": { + "properties": { + "ack": { + "type": "string" + }, + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/SyncEntityType" + } + ] + } + }, + "required": [ + "ack", + "type" + ], + "type": "object" + }, + "SyncAckSetDto": { + "properties": { + "acks": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "acks" + ], + "type": "object" + }, + "SyncEntityType": { + "enum": [ + "UserV1", + "UserDeleteV1" + ], + "type": "string" + }, + "SyncRequestType": { + "enum": [ + "UsersV1" + ], + "type": "string" + }, + "SyncStreamDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncRequestType" + }, + "type": "array" + } + }, + "required": [ + "types" + ], + "type": "object" + }, + "SyncUserDeleteV1": { + "properties": { + "userId": { + "type": "string" + } + }, + "required": [ + "userId" + ], + "type": "object" + }, + "SyncUserV1": { + "properties": { + "deletedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "deletedAt", + "email", + "id", + "name" + ], + "type": "object" + }, "SystemConfigBackupsDto": { "properties": { "database": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 9ff35331f..8b2e88183 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1104,6 +1104,16 @@ export type StackCreateDto = { export type StackUpdateDto = { primaryAssetId?: string; }; +export type SyncAckDeleteDto = { + types?: SyncEntityType[]; +}; +export type SyncAckDto = { + ack: string; + "type": SyncEntityType; +}; +export type SyncAckSetDto = { + acks: string[]; +}; export type AssetDeltaSyncDto = { updatedAfter: string; userIds: string[]; @@ -1119,6 +1129,9 @@ export type AssetFullSyncDto = { updatedUntil: string; userId?: string; }; +export type SyncStreamDto = { + types: SyncRequestType[]; +}; export type DatabaseBackupConfig = { cronExpression: string; enabled: boolean; @@ -2912,6 +2925,32 @@ export function updateStack({ id, stackUpdateDto }: { body: stackUpdateDto }))); } +export function deleteSyncAck({ syncAckDeleteDto }: { + syncAckDeleteDto: SyncAckDeleteDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "DELETE", + body: syncAckDeleteDto + }))); +} +export function getSyncAck(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: SyncAckDto[]; + }>("/sync/ack", { + ...opts + })); +} +export function sendSyncAck({ syncAckSetDto }: { + syncAckSetDto: SyncAckSetDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "POST", + body: syncAckSetDto + }))); +} export function getDeltaSync({ assetDeltaSyncDto }: { assetDeltaSyncDto: AssetDeltaSyncDto; }, opts?: Oazapfts.RequestOpts) { @@ -2936,6 +2975,15 @@ export function getFullSyncForUser({ assetFullSyncDto }: { body: assetFullSyncDto }))); } +export function getSyncStream({ syncStreamDto }: { + syncStreamDto: SyncStreamDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/stream", oazapfts.json({ + ...opts, + method: "POST", + body: syncStreamDto + }))); +} export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; @@ -3548,6 +3596,13 @@ export enum Error2 { NoPermission = "no_permission", NotFound = "not_found" } +export enum SyncEntityType { + UserV1 = "UserV1", + UserDeleteV1 = "UserDeleteV1" +} +export enum SyncRequestType { + UsersV1 = "UsersV1" +} export enum TranscodeHWAccel { Nvenc = "nvenc", Qsv = "qsv", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index a4518598a..b02d869a1 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -29,7 +29,7 @@ import { AuthService } from 'src/services/auth.service'; import { CliService } from 'src/services/cli.service'; import { DatabaseService } from 'src/services/database.service'; -const common = [...repositories, ...services]; +const common = [...repositories, ...services, GlobalExceptionFilter]; const middleware = [ FileUploadInterceptor, diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts index 4d970a710..0945810be 100644 --- a/server/src/controllers/sync.controller.ts +++ b/server/src/controllers/sync.controller.ts @@ -1,15 +1,28 @@ -import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Header, HttpCode, HttpStatus, Post, Res } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; +import { Response } from 'express'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; import { Auth, Authenticated } from 'src/middleware/auth.guard'; +import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter'; import { SyncService } from 'src/services/sync.service'; @ApiTags('Sync') @Controller('sync') export class SyncController { - constructor(private service: SyncService) {} + constructor( + private service: SyncService, + private errorService: GlobalExceptionFilter, + ) {} @Post('full-sync') @HttpCode(HttpStatus.OK) @@ -24,4 +37,37 @@ export class SyncController { getDeltaSync(@Auth() auth: AuthDto, @Body() dto: AssetDeltaSyncDto): Promise { return this.service.getDeltaSync(auth, dto); } + + @Post('stream') + @Header('Content-Type', 'application/jsonlines+json') + @HttpCode(HttpStatus.OK) + @Authenticated() + async getSyncStream(@Auth() auth: AuthDto, @Res() res: Response, @Body() dto: SyncStreamDto) { + try { + await this.service.stream(auth, res, dto); + } catch (error: Error | any) { + res.setHeader('Content-Type', 'application/json'); + this.errorService.handleError(res, error); + } + } + + @Get('ack') + @Authenticated() + getSyncAck(@Auth() auth: AuthDto): Promise { + return this.service.getAcks(auth); + } + + @Post('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + sendSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckSetDto) { + return this.service.setAcks(auth, dto); + } + + @Delete('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + deleteSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckDeleteDto) { + return this.service.deleteAcks(auth, dto); + } } diff --git a/server/src/database.ts b/server/src/database.ts index 4fcab0fd6..c3fb4cbab 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,3 +1,4 @@ +import { sql } from 'kysely'; import { Permission } from 'src/enum'; export type AuthUser = { @@ -29,6 +30,8 @@ export type AuthSession = { }; export const columns = { + ackEpoch: (columnName: 'createdAt' | 'updatedAt' | 'deletedAt') => + sql.raw(`extract(epoch from "${columnName}")::text`).as('ackEpoch'), authUser: [ 'users.id', 'users.name', diff --git a/server/src/db.d.ts b/server/src/db.d.ts index 2e10e1ade..255ac8cd2 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -4,7 +4,7 @@ */ import type { ColumnType } from 'kysely'; -import { Permission } from 'src/enum'; +import { Permission, SyncEntityType } from 'src/enum'; export type ArrayType = ArrayTypeImpl extends (infer U)[] ? U[] : ArrayTypeImpl; @@ -294,6 +294,15 @@ export interface Sessions { userId: string; } +export interface SessionSyncCheckpoints { + ack: string; + createdAt: Generated; + sessionId: string; + type: SyncEntityType; + updatedAt: Generated; +} + + export interface SharedLinkAsset { assetsId: string; sharedLinksId: string; @@ -384,6 +393,11 @@ export interface Users { updatedAt: Generated; } +export interface UsersAudit { + userId: string; + deletedAt: Generated; +} + export interface VectorsPgVectorIndexStat { idx_growing: ArrayType | null; idx_indexing: boolean | null; @@ -429,6 +443,7 @@ export interface DB { partners: Partners; person: Person; sessions: Sessions; + session_sync_checkpoints: SessionSyncCheckpoints; shared_link__asset: SharedLinkAsset; shared_links: SharedLinks; smart_search: SmartSearch; @@ -440,6 +455,7 @@ export interface DB { typeorm_metadata: TypeormMetadata; user_metadata: UserMetadata; users: Users; + users_audit: UsersAudit; 'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat; version_history: VersionHistory; } diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts index 820de8d6c..0628a566c 100644 --- a/server/src/dtos/sync.dto.ts +++ b/server/src/dtos/sync.dto.ts @@ -1,7 +1,8 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsInt, IsPositive } from 'class-validator'; +import { IsEnum, IsInt, IsPositive, IsString } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; -import { ValidateDate, ValidateUUID } from 'src/validation'; +import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { Optional, ValidateDate, ValidateUUID } from 'src/validation'; export class AssetFullSyncDto { @ValidateUUID({ optional: true }) @@ -32,3 +33,51 @@ export class AssetDeltaSyncResponseDto { upserted!: AssetResponseDto[]; deleted!: string[]; } + +export class SyncUserV1 { + id!: string; + name!: string; + email!: string; + deletedAt!: Date | null; +} + +export class SyncUserDeleteV1 { + userId!: string; +} + +export type SyncItem = { + [SyncEntityType.UserV1]: SyncUserV1; + [SyncEntityType.UserDeleteV1]: SyncUserDeleteV1; +}; + +const responseDtos = [ + // + SyncUserV1, + SyncUserDeleteV1, +]; + +export const extraSyncModels = responseDtos; + +export class SyncStreamDto { + @IsEnum(SyncRequestType, { each: true }) + @ApiProperty({ enumName: 'SyncRequestType', enum: SyncRequestType, isArray: true }) + types!: SyncRequestType[]; +} + +export class SyncAckDto { + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType }) + type!: SyncEntityType; + ack!: string; +} + +export class SyncAckSetDto { + @IsString({ each: true }) + acks!: string[]; +} + +export class SyncAckDeleteDto { + @IsEnum(SyncEntityType, { each: true }) + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType, isArray: true }) + @Optional() + types?: SyncEntityType[]; +} diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts index 75e92038a..a1df269c0 100644 --- a/server/src/entities/index.ts +++ b/server/src/entities/index.ts @@ -20,8 +20,10 @@ import { SessionEntity } from 'src/entities/session.entity'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SmartSearchEntity } from 'src/entities/smart-search.entity'; import { StackEntity } from 'src/entities/stack.entity'; +import { SessionSyncCheckpointEntity } from 'src/entities/sync-checkpoint.entity'; import { SystemMetadataEntity } from 'src/entities/system-metadata.entity'; import { TagEntity } from 'src/entities/tag.entity'; +import { UserAuditEntity } from 'src/entities/user-audit.entity'; import { UserMetadataEntity } from 'src/entities/user-metadata.entity'; import { UserEntity } from 'src/entities/user.entity'; import { VersionHistoryEntity } from 'src/entities/version-history.entity'; @@ -44,12 +46,14 @@ export const entities = [ MoveEntity, PartnerEntity, PersonEntity, + SessionSyncCheckpointEntity, SharedLinkEntity, SmartSearchEntity, StackEntity, SystemMetadataEntity, TagEntity, UserEntity, + UserAuditEntity, UserMetadataEntity, SessionEntity, LibraryEntity, diff --git a/server/src/entities/sync-checkpoint.entity.ts b/server/src/entities/sync-checkpoint.entity.ts new file mode 100644 index 000000000..2a91d2386 --- /dev/null +++ b/server/src/entities/sync-checkpoint.entity.ts @@ -0,0 +1,24 @@ +import { SessionEntity } from 'src/entities/session.entity'; +import { SyncEntityType } from 'src/enum'; +import { Column, CreateDateColumn, Entity, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('session_sync_checkpoints') +export class SessionSyncCheckpointEntity { + @ManyToOne(() => SessionEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + session?: SessionEntity; + + @PrimaryColumn() + sessionId!: string; + + @PrimaryColumn({ type: 'varchar' }) + type!: SyncEntityType; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt!: Date; + + @Column() + ack!: string; +} diff --git a/server/src/entities/user-audit.entity.ts b/server/src/entities/user-audit.entity.ts new file mode 100644 index 000000000..305994a6d --- /dev/null +++ b/server/src/entities/user-audit.entity.ts @@ -0,0 +1,14 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity('users_audit') +@Index('IDX_users_audit_deleted_at_asc_user_id_asc', ['deletedAt', 'userId']) +export class UserAuditEntity { + @PrimaryGeneratedColumn('increment') + id!: number; + + @Column({ type: 'uuid' }) + userId!: string; + + @CreateDateColumn({ type: 'timestamptz' }) + deletedAt!: Date; +} diff --git a/server/src/entities/user.entity.ts b/server/src/entities/user.entity.ts index 3f5b470ce..b597d15cf 100644 --- a/server/src/entities/user.entity.ts +++ b/server/src/entities/user.entity.ts @@ -10,12 +10,14 @@ import { CreateDateColumn, DeleteDateColumn, Entity, + Index, OneToMany, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm'; @Entity('users') +@Index('IDX_users_updated_at_asc_id_asc', ['updatedAt', 'id']) export class UserEntity { @PrimaryGeneratedColumn('uuid') id!: string; diff --git a/server/src/enum.ts b/server/src/enum.ts index 0c1fb01a1..b99518c4f 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -537,3 +537,12 @@ export enum DatabaseLock { GetSystemConfig = 69, BackupDatabase = 42, } + +export enum SyncRequestType { + UsersV1 = 'UsersV1', +} + +export enum SyncEntityType { + UserV1 = 'UserV1', + UserDeleteV1 = 'UserDeleteV1', +} diff --git a/server/src/middleware/global-exception.filter.ts b/server/src/middleware/global-exception.filter.ts index 7d7ade471..a8afa91cb 100644 --- a/server/src/middleware/global-exception.filter.ts +++ b/server/src/middleware/global-exception.filter.ts @@ -22,6 +22,13 @@ export class GlobalExceptionFilter implements ExceptionFilter { } } + handleError(res: Response, error: Error) { + const { status, body } = this.fromError(error); + if (!res.headersSent) { + res.status(status).json({ ...body, statusCode: status, correlationId: this.cls.getId() }); + } + } + private fromError(error: Error) { logGlobalError(this.logger, error); diff --git a/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts new file mode 100644 index 000000000..ef75dd7c0 --- /dev/null +++ b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddSessionSyncCheckpointTable1740001232576 implements MigrationInterface { + name = 'AddSessionSyncCheckpointTable1740001232576' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "ack" character varying NOT NULL, CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type"))`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(` + create trigger session_sync_checkpoints_updated_at + before update on session_sync_checkpoints + for each row execute procedure updated_at() + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`drop trigger session_sync_checkpoints_updated_at on session_sync_checkpoints`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" DROP CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc"`); + await queryRunner.query(`DROP TABLE "session_sync_checkpoints"`); + } + +} diff --git a/server/src/migrations/1740064899123-AddUsersAuditTable.ts b/server/src/migrations/1740064899123-AddUsersAuditTable.ts new file mode 100644 index 000000000..b8f2ce5e3 --- /dev/null +++ b/server/src/migrations/1740064899123-AddUsersAuditTable.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddUsersAuditTable1740064899123 implements MigrationInterface { + name = 'AddUsersAuditTable1740064899123' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt" ASC, "id" ASC);`) + await queryRunner.query(`CREATE TABLE "users_audit" ("id" SERIAL NOT NULL, "userId" uuid NOT NULL, "deletedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_audit_deleted_at_asc_user_id_asc" ON "users_audit" ("deletedAt" ASC, "userId" ASC);`) + await queryRunner.query(`CREATE OR REPLACE FUNCTION users_delete_audit() RETURNS TRIGGER AS + $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$ LANGUAGE plpgsql` + ); + await queryRunner.query(`CREATE OR REPLACE TRIGGER users_delete_audit + AFTER DELETE ON users + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION users_delete_audit(); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TRIGGER users_delete_audit`); + await queryRunner.query(`DROP FUNCTION users_delete_audit`); + await queryRunner.query(`DROP TABLE "users_audit"`); + } + +} diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index d3a8aeeb6..180d8ccd4 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -30,6 +30,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -71,6 +72,7 @@ export const repositories = [ SharedLinkRepository, StackRepository, StorageRepository, + SyncRepository, SystemMetadataRepository, TagRepository, TelemetryRepository, diff --git a/server/src/repositories/sync.repository.ts b/server/src/repositories/sync.repository.ts new file mode 100644 index 000000000..4023bf890 --- /dev/null +++ b/server/src/repositories/sync.repository.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { columns } from 'src/database'; +import { DB, SessionSyncCheckpoints } from 'src/db'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +@Injectable() +export class SyncRepository { + constructor(@InjectKysely() private db: Kysely) {} + + getCheckpoints(sessionId: string) { + return this.db + .selectFrom('session_sync_checkpoints') + .select(['type', 'ack']) + .where('sessionId', '=', sessionId) + .execute(); + } + + upsertCheckpoints(items: Insertable[]) { + return this.db + .insertInto('session_sync_checkpoints') + .values(items) + .onConflict((oc) => + oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({ + ack: eb.ref('excluded.ack'), + })), + ) + .execute(); + } + + deleteCheckpoints(sessionId: string, types?: SyncEntityType[]) { + return this.db + .deleteFrom('session_sync_checkpoints') + .where('sessionId', '=', sessionId) + .$if(!!types, (qb) => qb.where('type', 'in', types!)) + .execute(); + } + + getUserUpserts(ack?: SyncAck) { + return this.db + .selectFrom('users') + .select(['id', 'name', 'email', 'deletedAt']) + .select(columns.ackEpoch('updatedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('updatedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('updatedAt')), + eb('id', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['updatedAt asc', 'id asc']) + .stream(); + } + + getUserDeletes(ack?: SyncAck) { + return this.db + .selectFrom('users_audit') + .select(['userId']) + .select(columns.ackEpoch('deletedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('deletedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('deletedAt')), + eb('userId', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['deletedAt asc', 'userId asc']) + .stream(); + } +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index f476adba1..63cca43cc 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -38,6 +38,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -85,6 +86,7 @@ export class BaseService { protected sharedLinkRepository: SharedLinkRepository, protected stackRepository: StackRepository, protected storageRepository: StorageRepository, + protected syncRepository: SyncRepository, protected systemMetadataRepository: SystemMetadataRepository, protected tagRepository: TagRepository, protected telemetryRepository: TelemetryRepository, diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index fe967e37e..b94e8cfcb 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -1,18 +1,112 @@ -import { Injectable } from '@nestjs/common'; +import { ForbiddenException, Injectable } from '@nestjs/common'; +import { Insertable } from 'kysely'; import { DateTime } from 'luxon'; +import { Writable } from 'node:stream'; import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; +import { SessionSyncCheckpoints } from 'src/db'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; -import { DatabaseAction, EntityType, Permission } from 'src/enum'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; +import { DatabaseAction, EntityType, Permission, SyncEntityType, SyncRequestType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; +import { SyncAck } from 'src/types'; import { getMyPartnerIds } from 'src/utils/asset.util'; import { setIsEqual } from 'src/utils/set'; +import { fromAck, serialize } from 'src/utils/sync'; const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] }; +const SYNC_TYPES_ORDER = [ + // + SyncRequestType.UsersV1, +]; + +const throwSessionRequired = () => { + throw new ForbiddenException('Sync endpoints cannot be used with API keys'); +}; @Injectable() export class SyncService extends BaseService { + getAcks(auth: AuthDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + return this.syncRepository.getCheckpoints(sessionId); + } + + async setAcks(auth: AuthDto, dto: SyncAckSetDto) { + // TODO ack validation + + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints: Insertable[] = []; + for (const ack of dto.acks) { + const { type } = fromAck(ack); + checkpoints.push({ sessionId, type, ack }); + } + + await this.syncRepository.upsertCheckpoints(checkpoints); + } + + async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + await this.syncRepository.deleteCheckpoints(sessionId, dto.types); + } + + async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints = await this.syncRepository.getCheckpoints(sessionId); + const checkpointMap: Partial> = Object.fromEntries( + checkpoints.map(({ type, ack }) => [type, fromAck(ack)]), + ); + + // TODO pre-filter/sort list based on optimal sync order + + for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) { + switch (type) { + case SyncRequestType.UsersV1: { + const deletes = this.syncRepository.getUserDeletes(checkpointMap[SyncEntityType.UserDeleteV1]); + for await (const { ackEpoch, ...data } of deletes) { + response.write(serialize({ type: SyncEntityType.UserDeleteV1, ackEpoch, ids: [data.userId], data })); + } + + const upserts = this.syncRepository.getUserUpserts(checkpointMap[SyncEntityType.UserV1]); + for await (const { ackEpoch, ...data } of upserts) { + response.write(serialize({ type: SyncEntityType.UserV1, ackEpoch, ids: [data.id], data })); + } + + break; + } + + default: { + this.logger.warn(`Unsupported sync type: ${type}`); + break; + } + } + } + + response.end(); + } + async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { // mobile implementation is faster if this is a single id const userId = dto.userId || auth.user.id; diff --git a/server/src/types.ts b/server/src/types.ts index 3a331127e..544d35524 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -4,6 +4,7 @@ import { ImageFormat, JobName, QueueName, + SyncEntityType, TranscodeTarget, VideoCodec, } from 'src/enum'; @@ -409,3 +410,9 @@ export interface IBulkAsset { addAssetIds: (id: string, assetIds: string[]) => Promise; removeAssetIds: (id: string, assetIds: string[]) => Promise; } + +export type SyncAck = { + type: SyncEntityType; + ackEpoch: string; + ids: string[]; +}; diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 13969543e..e07d0fe03 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -12,6 +12,7 @@ import { writeFileSync } from 'node:fs'; import path from 'node:path'; import { SystemConfig } from 'src/config'; import { CLIP_MODEL_INFO, serverVersion } from 'src/constants'; +import { extraSyncModels } from 'src/dtos/sync.dto'; import { ImmichCookie, ImmichHeader, MetadataKey } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -245,6 +246,7 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean }) const options: SwaggerDocumentOptions = { operationIdFactory: (controllerKey: string, methodKey: string) => methodKey, + extraModels: extraSyncModels, }; const specification = SwaggerModule.createDocument(app, config, options); diff --git a/server/src/utils/sync.ts b/server/src/utils/sync.ts new file mode 100644 index 000000000..8e426ab86 --- /dev/null +++ b/server/src/utils/sync.ts @@ -0,0 +1,30 @@ +import { SyncItem } from 'src/dtos/sync.dto'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +type Impossible = { + [P in K]: never; +}; + +type Exact = U & Impossible>; + +export const fromAck = (ack: string): SyncAck => { + const [type, timestamp, ...ids] = ack.split('|'); + return { type: type as SyncEntityType, ackEpoch: timestamp, ids }; +}; + +export const toAck = ({ type, ackEpoch, ids }: SyncAck) => [type, ackEpoch, ...ids].join('|'); + +export const mapJsonLine = (object: unknown) => JSON.stringify(object) + '\n'; + +export const serialize = ({ + type, + ackEpoch, + ids, + data, +}: { + type: T; + ackEpoch: string; + ids: string[]; + data: Exact; +}) => mapJsonLine({ type, data, ack: toAck({ type, ackEpoch, ids }) }); diff --git a/server/test/repositories/sync.repository.mock.ts b/server/test/repositories/sync.repository.mock.ts new file mode 100644 index 000000000..fbb8ec2f6 --- /dev/null +++ b/server/test/repositories/sync.repository.mock.ts @@ -0,0 +1,13 @@ +import { SyncRepository } from 'src/repositories/sync.repository'; +import { RepositoryInterface } from 'src/types'; +import { Mocked, vitest } from 'vitest'; + +export const newSyncRepositoryMock = (): Mocked> => { + return { + getCheckpoints: vitest.fn(), + upsertCheckpoints: vitest.fn(), + deleteCheckpoints: vitest.fn(), + getUserUpserts: vitest.fn(), + getUserDeletes: vitest.fn(), + }; +}; diff --git a/server/test/utils.ts b/server/test/utils.ts index d1dda3eed..ca2272f6b 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -34,6 +34,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -75,6 +76,7 @@ import { newSessionRepositoryMock } from 'test/repositories/session.repository.m import { newSharedLinkRepositoryMock } from 'test/repositories/shared-link.repository.mock'; import { newStackRepositoryMock } from 'test/repositories/stack.repository.mock'; import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; +import { newSyncRepositoryMock } from 'test/repositories/sync.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { newTagRepositoryMock } from 'test/repositories/tag.repository.mock'; import { ITelemetryRepositoryMock, newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; @@ -178,6 +180,7 @@ export const newTestService = ( const sharedLinkMock = newSharedLinkRepositoryMock(); const stackMock = newStackRepositoryMock(); const storageMock = newStorageRepositoryMock(); + const syncMock = newSyncRepositoryMock(); const systemMock = newSystemMetadataRepositoryMock(); const tagMock = newTagRepositoryMock(); const telemetryMock = newTelemetryRepositoryMock(); @@ -219,6 +222,7 @@ export const newTestService = ( sharedLinkMock as RepositoryInterface as SharedLinkRepository, stackMock as RepositoryInterface as StackRepository, storageMock as RepositoryInterface as StorageRepository, + syncMock as RepositoryInterface as SyncRepository, systemMock as RepositoryInterface as SystemMetadataRepository, tagMock as RepositoryInterface as TagRepository, telemetryMock as unknown as TelemetryRepository,