diff --git a/.gitignore b/.gitignore index c3e2064caad..63de418bdcc 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ openmetadata-ui/src/main/resources/ui/.env openmetadata-ui/src/main/resources/ui/playwright/.auth openmetadata-ui/src/main/resources/ui/blob-report openmetadata-ui/src/main/resources/ui/test-results/ +openmetadata-ui/src/main/resources/ui/debug.json #UI - Dereferenced Schemas openmetadata-ui/src/main/resources/ui/src/jsons/* diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java index 4c3ad19379e..ccda42053f2 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java @@ -5692,6 +5692,45 @@ public abstract class BaseEntityIT { } } + /** + * Test: CSV import/export round-trip preserves Unicode text. + * Verifies non-ASCII content survives export, import, and re-export. + */ + @Test + void test_importExportRoundTripUnicode(TestNamespace ns) { + Assumptions.assumeTrue(supportsImportExport, "Entity does not support import/export"); + + org.openmetadata.sdk.services.EntityServiceBase service = getEntityService(); + Assumptions.assumeTrue(service != null, "Entity service not provided"); + + String containerName = getImportExportContainerName(ns); + Assumptions.assumeTrue(containerName != null, "Container name not provided"); + + K createRequest = createMinimalRequest(ns); + String unicodeDescription = "中文描述 - CSV import/export round trip"; + setDescription(createRequest, unicodeDescription); + T entity = createEntity(createRequest); + assertNotNull(entity, "Entity should be created"); + + try { + String exportedCsv = service.exportCsv(containerName); + assertNotNull(exportedCsv, "Export should return CSV data"); + assertTrue( + exportedCsv.contains(unicodeDescription), "Exported CSV should contain Unicode text"); + + CsvImportResult importResult = performImportCsv(ns, exportedCsv, false); + assertEquals( + ApiStatus.SUCCESS, importResult.getStatus(), "Unicode round-trip should succeed"); + + String reExportedCsv = service.exportCsv(containerName); + assertNotNull(reExportedCsv, "Re-export should return CSV data"); + assertTrue( + reExportedCsv.contains(unicodeDescription), "Re-exported CSV should retain Unicode text"); + } catch (Exception e) { + fail("Unicode import/export round-trip failed", e); + } + } + // =================================================================== // COMPREHENSIVE CSV IMPORT/EXPORT TESTS // Template-based tests that work with any entity CSV structure diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 9c1199a2955..ead2fb57059 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -40,6 +40,7 @@ import org.openmetadata.schema.type.csv.CsvHeader; public final class CsvUtil { public static final String SEPARATOR = ","; public static final String FIELD_SEPARATOR = ";"; + public static final String UTF8_BOM = "\uFEFF"; public static final String ENTITY_TYPE_SEPARATOR = ":"; public static final String LINE_SEPARATOR = "\r\n"; @@ -50,6 +51,20 @@ public final class CsvUtil { // Utility class hides the constructor } + public static String stripUtf8Bom(String value) { + if (value == null || value.isEmpty() || !value.startsWith(UTF8_BOM)) { + return value; + } + return value.substring(1); + } + + public static String withUtf8Bom(String value) { + if (value == null || value.startsWith(UTF8_BOM)) { + return value; + } + return UTF8_BOM + value; + } + public static String formatCsv(CsvFile csvFile) throws IOException { // CSV file is generated by the backend and the data exported is expected to be correct. Hence, // no validation diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java index bc75c9b27b5..12b6153b27e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java @@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.csv.CsvUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.data.BulkColumnUpdatePreview; import org.openmetadata.schema.api.data.BulkColumnUpdateRequest; @@ -875,7 +876,7 @@ public class ColumnRepository { result.setNumberOfRowsPassed(0); result.setNumberOfRowsFailed(0); - String[] lines = csv.split("\n"); + String[] lines = CsvUtil.stripUtf8Bom(csv).split("\n"); if (lines.length <= 1) { result.setStatus(ApiStatus.ABORTED); result.setAbortReason("No data to import"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index 57f62d757e5..140a768a97d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -53,6 +53,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.csv.CsvExportProgressCallback; import org.openmetadata.csv.CsvImportProgressCallback; +import org.openmetadata.csv.CsvUtil; import org.openmetadata.schema.BulkAssetsRequestInterface; import org.openmetadata.schema.CreateEntity; import org.openmetadata.schema.EntityInterface; @@ -929,11 +930,12 @@ public abstract class EntityResource { @GET @Path("/name/{name}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( operationId = "exportTable", summary = "Export table in CSV format", responses = { @ApiResponse( - responseCode = "200", - description = "Exported csv with columns from the table", + responseCode = "202", + description = "Export initiated successfully", content = @Content( mediaType = "application/json", - schema = @Schema(implementation = String.class))) + schema = @Schema(implementation = CSVExportResponse.class))) }) public Response exportCsvAsync( @Context SecurityContext securityContext, @@ -606,7 +607,7 @@ public class TableResource extends EntityResource { @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({"text/csv; charset=UTF-8"}) @Valid @Operation( operationId = "exportTable", @@ -617,7 +618,7 @@ public class TableResource extends EntityResource { description = "Exported csv with columns from the table", content = @Content( - mediaType = "application/json", + mediaType = "text/csv; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportCsv( @@ -631,7 +632,7 @@ public class TableResource extends EntityResource { @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTable", @@ -665,7 +666,7 @@ public class TableResource extends EntityResource { @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTableAsync", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index f5140b49345..7fa6cc601fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -1252,7 +1252,7 @@ public class TestCaseResource extends EntityResource { @GET @Path("/name/{name}/exportAsync") - @Produces(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( operationId = "exportTeams", summary = "Export teams in CSV format", responses = { @ApiResponse( - responseCode = "200", - description = "Exported csv with teams information", + responseCode = "202", + description = "Export initiated successfully", content = @Content( mediaType = "application/json", @@ -740,7 +740,7 @@ public class TeamResource extends EntityResource { @GET @Path("/name/{name}/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportTeams", @@ -751,7 +751,7 @@ public class TeamResource extends EntityResource { description = "Exported csv with teams information", content = @Content( - mediaType = "application/json", + mediaType = "text/plain; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name) @@ -761,7 +761,7 @@ public class TeamResource extends EntityResource { @PUT @Path("/name/{name}/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "importTeams", @@ -854,7 +854,7 @@ public class TeamResource extends EntityResource { @PUT @Path("/name/{name}/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Produces(MediaType.APPLICATION_JSON) @Valid @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java index 552efc93e37..4c38a23b843 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java @@ -1688,7 +1688,7 @@ public class UserResource extends EntityResource { @GET @Path("/export") - @Produces(MediaType.TEXT_PLAIN) + @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( operationId = "exportUsers", @@ -1699,7 +1699,7 @@ public class UserResource extends EntityResource { description = "Exported csv with user information", content = @Content( - mediaType = "application/json", + mediaType = "text/plain; charset=UTF-8", schema = @Schema(implementation = String.class))) }) public String exportUsersCsv( @@ -1716,11 +1716,11 @@ public class UserResource extends EntityResource { @PUT @Path("/import") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( - operationId = "importTeams", - summary = "Import from CSV to create, and update teams.", + operationId = "importUsers", + summary = "Import from CSV to create, and update users.", responses = { @ApiResponse( responseCode = "200", @@ -1753,11 +1753,11 @@ public class UserResource extends EntityResource { @PUT @Path("/importAsync") - @Consumes(MediaType.TEXT_PLAIN) + @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"}) @Valid @Operation( - operationId = "importTeamsAsync", - summary = "Import from CSV to create, and update teams asynchronously.", + operationId = "importUsersAsync", + summary = "Import from CSV to create, and update users asynchronously.", responses = { @ApiResponse( responseCode = "200", diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java index 677882802a4..09fe4a86150 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java @@ -32,6 +32,8 @@ import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TermRelation; +import org.openmetadata.schema.type.csv.CsvFile; +import org.openmetadata.schema.type.csv.CsvHeader; public class CsvUtilTest { @Test @@ -322,4 +324,41 @@ public class CsvUtilTest { assertEquals(expectedCsvRecords.get(i), actualCsvRecords.get(i)); } } + + @Test + void testFormatCsvPreservesChineseCharacters() throws Exception { + CsvFile csvFile = + new CsvFile() + .withHeaders( + List.of( + new CsvHeader().withName("name"), + new CsvHeader().withName("description"), + new CsvHeader().withName("owner"))) + .withRecords(List.of(List.of("中文表", "这是中文描述", "数据平台团队"))); + + String csv = CsvUtil.formatCsv(csvFile); + + assertTrue(csv.contains("中文表")); + assertTrue(csv.contains("这是中文描述")); + assertTrue(csv.contains("数据平台团队")); + } + + @Test + void testStripUtf8Bom() { + String csv = "name,description\n中文表,中文描述"; + assertEquals(csv, CsvUtil.stripUtf8Bom(CsvUtil.UTF8_BOM + csv)); + assertEquals(csv, CsvUtil.stripUtf8Bom(csv)); + assertEquals("", CsvUtil.stripUtf8Bom("")); + assertNull(CsvUtil.stripUtf8Bom(null)); + } + + @Test + void testWithUtf8Bom() { + String csv = "name,description\nunicode-name,unicode-description"; + + assertEquals(CsvUtil.UTF8_BOM + csv, CsvUtil.withUtf8Bom(csv)); + assertEquals(CsvUtil.UTF8_BOM + csv, CsvUtil.withUtf8Bom(CsvUtil.UTF8_BOM + csv)); + assertEquals(CsvUtil.UTF8_BOM, CsvUtil.withUtf8Bom("")); + assertNull(CsvUtil.withUtf8Bom(null)); + } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index 0b31ada1ad1..5e805bd1e2d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -63,6 +63,13 @@ const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES); const propertyListName: Record = {}; const additionalGlossaryTerm = createGlossaryTermRowDetails(); +const chineseGlossaryTermDetails = { + name: `术语${uuid()}`, + displayName: '中文术语展示名', + description: '这是用于验证导入导出编码的中文描述。', + synonyms: '中文同义词;测试', + references: '参考;https://example.com/%E4%B8%AD%E6%96%87', +}; test.describe('Glossary Bulk Import Export', () => { test.slow(true); @@ -183,6 +190,7 @@ test.describe('Glossary Bulk Import Export', () => { await fillGlossaryRowDetails( { ...additionalGlossaryTerm, + ...chineseGlossaryTermDetails, owners: [user1.responseData?.['displayName']], reviewers: [user2.responseData?.['displayName']], relatedTerm: { @@ -214,6 +222,9 @@ test.describe('Glossary Bulk Import Export', () => { const rowStatus = ['Entity updated', 'Entity created']; await expect(page.locator('.rdg-cell-details')).toHaveText(rowStatus); + await expect( + page.getByText(chineseGlossaryTermDetails.name) + ).toBeVisible(); await page.getByRole('button', { name: 'Update' }).click(); await page diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts index 835d27fb077..a267d7394e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts @@ -21,5 +21,5 @@ export interface UploadFileProps { file: RcFile, FileList: RcFile[] ) => BeforeUploadValueType | Promise; - onCSVUploaded: (event: ProgressEvent) => void; + onCSVUploaded: (event: ProgressEvent) => void | Promise; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.test.tsx index 39e61c00148..c84698951a9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.test.tsx @@ -34,7 +34,14 @@ describe('UploadFile Component', () => { onCSVUploaded: jest.fn(), }; + let originalFileReader: typeof FileReader; + + beforeEach(() => { + originalFileReader = global.FileReader; + }); + afterEach(() => { + global.FileReader = originalFileReader; jest.clearAllMocks(); }); @@ -74,6 +81,16 @@ describe('UploadFile Component', () => { it('should call onCSVUploaded when file is uploaded successfully', async () => { const mockOnCSVUploaded = jest.fn(); + const readAsText = jest.fn(function (this: FileReader) { + this.onload?.({ target: this } as ProgressEvent); + }); + const MockFileReader = jest.fn().mockImplementation(() => ({ + error: null, + onerror: null, + onload: null, + readAsText, + })); + global.FileReader = MockFileReader as unknown as typeof FileReader; render(); @@ -91,6 +108,8 @@ describe('UploadFile Component', () => { await waitFor(() => { expect(mockOnCSVUploaded).toHaveBeenCalled(); }); + + expect(readAsText).toHaveBeenCalledWith(file, 'utf-8'); }); it('should handle file upload error', async () => { @@ -106,13 +125,12 @@ describe('UploadFile Component', () => { }); // Mock FileReader to throw an error - const originalFileReader = global.FileReader; global.FileReader = jest.fn().mockImplementation(() => ({ readAsText: jest.fn(() => { throw new Error('File read error'); }), onerror: null, - })) as any; + })) as unknown as typeof FileReader; fireEvent.drop(uploadWidget, { dataTransfer: { @@ -123,9 +141,40 @@ describe('UploadFile Component', () => { await waitFor(() => { expect(showErrorToast).toHaveBeenCalled(); }); + }); - // Restore original FileReader - global.FileReader = originalFileReader; + it('should show error toast when onCSVUploaded rejects', async () => { + const error = new Error('Upload failed'); + const mockOnCSVUploaded = jest.fn().mockRejectedValue(error); + const readAsText = jest.fn(function (this: FileReader) { + this.onload?.({ target: this } as ProgressEvent); + }); + const MockFileReader = jest.fn().mockImplementation(() => ({ + error: null, + onerror: null, + onload: null, + readAsText, + })); + global.FileReader = MockFileReader as unknown as typeof FileReader; + + render(); + + const uploadWidget = screen.getByTestId('upload-file-widget'); + const file = new File(['test,csv,content'], 'test.csv', { + type: 'text/csv', + }); + + fireEvent.drop(uploadWidget, { + dataTransfer: { + files: [file], + }, + }); + + await waitFor(() => { + expect(showErrorToast).toHaveBeenCalledWith(error); + }); + + expect(readAsText).toHaveBeenCalledWith(file, 'utf-8'); }); it('should call beforeUpload when provided', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx index b9a69089435..0bff22a0220 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx @@ -35,17 +35,22 @@ const UploadFile: FC = ({ const handleUpload: UploadProps['customRequest'] = useCallback( (options: UploadRequestOption) => { setUploading(true); - const reader = new FileReader(); - reader.onload = (e) => { - setUploading(false); - onCSVUploaded(e); - }; - reader.onerror = () => { - setUploading(false); - showErrorToast(new Error(t('server.unexpected-error')) as AxiosError); - }; try { - reader.readAsText(options.file as Blob); + const reader = new FileReader(); + reader.onload = async (event) => { + try { + await onCSVUploaded(event); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setUploading(false); + } + }; + reader.onerror = () => { + showErrorToast(reader.error?.message ?? t('server.unexpected-error')); + setUploading(false); + }; + reader.readAsText(options.file as Blob, 'utf-8'); } catch (error) { setUploading(false); showErrorToast(error as AxiosError); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts index 55435953b56..19a87e0f037 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts @@ -166,7 +166,7 @@ export const importColumnsCSV = async ( `/columns/import?${queryParams.toString()}`, params.csv, { - headers: { 'Content-Type': 'text/plain' }, + headers: { 'Content-Type': 'text/plain; charset=UTF-8' }, } ); @@ -198,7 +198,7 @@ export const importColumnsCSVAsync = async ( string, AxiosResponse >(`/columns/import-async?${queryParams.toString()}`, params.csv, { - headers: { 'Content-Type': 'text/plain' }, + headers: { 'Content-Type': 'text/plain; charset=UTF-8' }, }); return response.data; diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts index 0f0a570e46b..6320d109e41 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts @@ -307,7 +307,7 @@ export const importDatabaseInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/databases/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, @@ -340,7 +340,7 @@ export const importDatabaseSchemaInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/databaseSchemas/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts index 4dde904bd6e..b5ce076e950 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts @@ -50,7 +50,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -78,7 +78,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=false&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -105,7 +105,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -132,7 +132,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/dataQuality/testCases/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -177,7 +177,7 @@ describe('importExportAPI tests', () => { }); expect(mockPut).toHaveBeenCalledWith(expect.any(String), '', { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }); }); }); @@ -205,7 +205,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/tables/name/database.schema.table/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -232,7 +232,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/databases/name/service.database/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -258,7 +258,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/databaseSchemas/name/service.database.schema/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -286,7 +286,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/tables/name/database.schema.table/importAsync?dryRun=false&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -312,7 +312,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.any(String), expect.any(String), - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -361,7 +361,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/databaseServices/name/my-database-service/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -388,7 +388,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/messagingServices/name/my-messaging-service/importAsync?dryRun=true&recursive=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -416,7 +416,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( 'services/databaseServices/name/my-service/importAsync?dryRun=false&recursive=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -465,7 +465,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaries/name/my-glossary/importAsync?dryRun=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -493,7 +493,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaries/name/my-glossary/importAsync?dryRun=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -547,7 +547,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/glossaries/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -596,7 +596,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaryTerms/name/glossary.term/importAsync?dryRun=true', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); expect(result).toEqual(mockCSVImportResponse); }); @@ -624,7 +624,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( '/glossaryTerms/name/glossary.term/importAsync?dryRun=false', mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -678,7 +678,7 @@ describe('importExportAPI tests', () => { expect(mockPut).toHaveBeenCalledWith( expect.stringContaining('/glossaryTerms/name/'), mockCSVData, - { headers: { 'Content-type': 'text/plain' } } + { headers: { 'Content-type': 'text/plain; charset=UTF-8' } } ); }); @@ -724,7 +724,7 @@ describe('importExportAPI tests', () => { }); expect(mockPut).toHaveBeenCalledWith(expect.any(String), largeCsvData, { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts index 10714ae70a6..0d8e8e8e13d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts @@ -33,7 +33,7 @@ export const importTestCaseInCSVFormat = async ({ targetEntityType, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; let url = `/dataQuality/testCases/name/${getEncodedFqn( name @@ -58,7 +58,7 @@ export const importEntityInCSVFormat = async ({ recursive = false, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put< string, @@ -82,7 +82,7 @@ export const importServiceInCSVFormat = async ({ recursive = false, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put< string, @@ -104,7 +104,7 @@ export const importGlossaryInCSVFormat = async ({ dryRun = true, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const response = await APIClient.put< string, @@ -124,7 +124,7 @@ export const importGlossaryTermInCSVFormat = async ({ dryRun = true, }: importEntityInCSVFormatRequestParams) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const response = await APIClient.put< string, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts index c66fff88661..265450e79e3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts @@ -264,7 +264,7 @@ export const importTableInCSVFormat = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, }; const res = await APIClient.put( `/tables/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts index a8c9b75c772..7b85d6910b3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts @@ -130,7 +130,7 @@ export const importTeam = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, params: { dryRun, }, @@ -149,7 +149,7 @@ export const importUserInTeam = async ( dryRun = true ) => { const configOptions = { - headers: { 'Content-type': 'text/plain' }, + headers: { 'Content-type': 'text/plain; charset=UTF-8' }, params: { team, dryRun, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx index 2a2146f577f..0919b247112 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx @@ -29,6 +29,10 @@ jest.mock('../ToastUtils', () => ({ })); describe('ExportUtils', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + describe('downloadFile', () => { const mockLink = { href: '', @@ -36,6 +40,9 @@ describe('ExportUtils', () => { style: { visibility: '' }, click: jest.fn(), }; + const originalBlob = global.Blob; + const originalCreateObjectURL = global.URL.createObjectURL; + const originalRevokeObjectURL = global.URL.revokeObjectURL; let mockCreateObjectURL: jest.Mock; let mockRevokeObjectURL: jest.Mock; @@ -57,7 +64,9 @@ describe('ExportUtils', () => { }); afterEach(() => { - jest.restoreAllMocks(); + global.Blob = originalBlob; + global.URL.createObjectURL = originalCreateObjectURL; + global.URL.revokeObjectURL = originalRevokeObjectURL; }); it('creates an anchor element and triggers a click', () => { @@ -92,15 +101,39 @@ describe('ExportUtils', () => { expect(mockRevokeObjectURL).toHaveBeenCalledWith('blob:mock-url'); }); - it('uses the provided mimeType when creating the Blob', () => { + it('normalizes CSV mimeType and prepends BOM when creating the Blob', () => { const mockBlob = {}; const MockBlob = jest.fn().mockReturnValue(mockBlob); global.Blob = MockBlob as unknown as typeof Blob; downloadFile('content', 'file.csv', 'text/csv;charset=utf-8;'); + expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], { + type: 'text/csv; charset=utf-8', + }); + }); + + it('does not prepend a duplicate BOM when content already has one', () => { + const mockBlob = {}; + const MockBlob = jest.fn().mockReturnValue(mockBlob); + global.Blob = MockBlob as unknown as typeof Blob; + + downloadFile('\uFEFFcontent', 'file.csv', 'text/csv;charset=utf-8;'); + + expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], { + type: 'text/csv; charset=utf-8', + }); + }); + + it('does not prepend BOM for non-csv files', () => { + const mockBlob = {}; + const MockBlob = jest.fn().mockReturnValue(mockBlob); + global.Blob = MockBlob as unknown as typeof Blob; + + downloadFile('content', 'file.txt'); + expect(MockBlob).toHaveBeenCalledWith(['content'], { - type: 'text/csv;charset=utf-8;', + type: 'text/plain', }); }); }); @@ -121,10 +154,6 @@ describe('ExportUtils', () => { } as unknown as HTMLAnchorElement); }); - afterEach(() => { - mockCreateElement.mockRestore(); - }); - it('should create and trigger download with correct attributes', () => { const dataUrl = 'data:image/png;base64,test'; const fileName = 'test-image'; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts index e7bbe3e57d1..ad2509336db 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts @@ -23,7 +23,17 @@ export const downloadFile = ( fileName: string, mimeType: string = 'text/plain' ): void => { - const blob = new Blob([content], { type: mimeType }); + const isCsvFile = fileName.toLowerCase().endsWith('.csv'); + const isCsvMime = mimeType.toLowerCase().includes('text/csv'); + const csvMimeType = 'text/csv; charset=utf-8'; + const effectiveMimeType = isCsvFile || isCsvMime ? csvMimeType : mimeType; + const effectiveContent = + isCsvFile || isCsvMime + ? content.startsWith('\uFEFF') + ? content + : `\uFEFF${content}` + : content; + const blob = new Blob([effectiveContent], { type: effectiveMimeType }); const link = document.createElement('a'); link.href = URL.createObjectURL(blob);