Elgato_dark/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata synced 2026-05-24 09:39:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge dc81ed20e5 into 42881cc0fc

Browse source
This commit is contained in:
Darshan Rajput 2026-05-24 09:24:48 +05:30 committed by GitHub
commit 8620c0969e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 332 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -92,6 +92,7 @@ openmetadata-ui/src/main/resources/ui/.env
openmetadata-ui/src/main/resources/ui/playwright/.auth
openmetadata-ui/src/main/resources/ui/blob-report
openmetadata-ui/src/main/resources/ui/test-results/
openmetadata-ui/src/main/resources/ui/debug.json
#UI - Dereferenced Schemas
openmetadata-ui/src/main/resources/ui/src/jsons/*

39
openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java
View file

@ -5692,6 +5692,45 @@ public abstract class BaseEntityIT<T extends EntityInterface, K> {
}
}
/**
* Test: CSV import/export round-trip preserves Unicode text.
* Verifies non-ASCII content survives export, import, and re-export.
*/
@Test
void test_importExportRoundTripUnicode(TestNamespace ns) {
Assumptions.assumeTrue(supportsImportExport, "Entity does not support import/export");
org.openmetadata.sdk.services.EntityServiceBase<T> service = getEntityService();
Assumptions.assumeTrue(service != null, "Entity service not provided");
String containerName = getImportExportContainerName(ns);
Assumptions.assumeTrue(containerName != null, "Container name not provided");
K createRequest = createMinimalRequest(ns);
String unicodeDescription = "中文描述 - CSV import/export round trip";
setDescription(createRequest, unicodeDescription);
T entity = createEntity(createRequest);
assertNotNull(entity, "Entity should be created");
try {
String exportedCsv = service.exportCsv(containerName);
assertNotNull(exportedCsv, "Export should return CSV data");
assertTrue(
exportedCsv.contains(unicodeDescription), "Exported CSV should contain Unicode text");
CsvImportResult importResult = performImportCsv(ns, exportedCsv, false);
assertEquals(
ApiStatus.SUCCESS, importResult.getStatus(), "Unicode round-trip should succeed");
String reExportedCsv = service.exportCsv(containerName);
assertNotNull(reExportedCsv, "Re-export should return CSV data");
assertTrue(
reExportedCsv.contains(unicodeDescription), "Re-exported CSV should retain Unicode text");
} catch (Exception e) {
fail("Unicode import/export round-trip failed", e);
}
}
// ===================================================================
// COMPREHENSIVE CSV IMPORT/EXPORT TESTS
// Template-based tests that work with any entity CSV structure

15
openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
View file

@ -40,6 +40,7 @@ import org.openmetadata.schema.type.csv.CsvHeader;
public final class CsvUtil {
public static final String SEPARATOR = ",";
public static final String FIELD_SEPARATOR = ";";
public static final String UTF8_BOM = "\uFEFF";
public static final String ENTITY_TYPE_SEPARATOR = ":";
public static final String LINE_SEPARATOR = "\r\n";
@ -50,6 +51,20 @@ public final class CsvUtil {
// Utility class hides the constructor
}
public static String stripUtf8Bom(String value) {
if (value == null || value.isEmpty() || !value.startsWith(UTF8_BOM)) {
return value;
}
return value.substring(1);
}
public static String withUtf8Bom(String value) {
if (value == null || value.startsWith(UTF8_BOM)) {
return value;
}
return UTF8_BOM + value;
}
public static String formatCsv(CsvFile csvFile) throws IOException {
// CSV file is generated by the backend and the data exported is expected to be correct. Hence,
// no validation

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java
View file

@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.csv.CsvUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.api.data.BulkColumnUpdatePreview;
import org.openmetadata.schema.api.data.BulkColumnUpdateRequest;
@ -875,7 +876,7 @@ public class ColumnRepository {
result.setNumberOfRowsPassed(0);
result.setNumberOfRowsFailed(0);
String[] lines = csv.split("\n");
String[] lines = CsvUtil.stripUtf8Bom(csv).split("\n");
if (lines.length <= 1) {
result.setStatus(ApiStatus.ABORTED);
result.setAbortReason("No data to import");

20
openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java
View file

@ -53,6 +53,7 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.csv.CsvExportProgressCallback;
import org.openmetadata.csv.CsvImportProgressCallback;
import org.openmetadata.csv.CsvUtil;
import org.openmetadata.schema.BulkAssetsRequestInterface;
import org.openmetadata.schema.CreateEntity;
import org.openmetadata.schema.EntityInterface;
@ -929,11 +930,12 @@ public abstract class EntityResource<T extends EntityInterface, K extends Entity
jobId, securityContext, exported, total, message);
String csvData =
repository.exportToCsv(
name,
securityContext.getUserPrincipal().getName(),
recursive,
progressCallback);
CsvUtil.withUtf8Bom(
repository.exportToCsv(
name,
securityContext.getUserPrincipal().getName(),
recursive,
progressCallback));
WebsocketNotificationHandler.sendCsvExportCompleteNotification(
jobId, securityContext, csvData);
} catch (Exception e) {
@ -1116,7 +1118,8 @@ public abstract class EntityResource<T extends EntityInterface, K extends Entity
OperationContext operationContext =
new OperationContext(entityType, MetadataOperation.VIEW_ALL);
authorizer.authorize(securityContext, operationContext, getResourceContextByName(name));
return repository.exportToCsv(name, securityContext.getUserPrincipal().getName(), recursive);
return CsvUtil.withUtf8Bom(
repository.exportToCsv(name, securityContext.getUserPrincipal().getName(), recursive));
}
protected CsvImportResult importCsvInternal(
@ -1156,18 +1159,19 @@ public abstract class EntityResource<T extends EntityInterface, K extends Entity
OperationContext operationContext =
new OperationContext(entityType, MetadataOperation.EDIT_ALL);
authorizer.authorize(securityContext, operationContext, getResourceContextByName(name));
String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
CsvImportResult result =
nullOrEmpty(versioningEntityType)
? repository.importFromCsv(
name,
csv,
normalizedCsv,
dryRun,
securityContext.getUserPrincipal().getName(),
recursive,
progressCallback)
: repository.importFromCsv(
name,
csv,
normalizedCsv,
dryRun,
securityContext.getUserPrincipal().getName(),
recursive,

26
openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java
View file

@ -40,6 +40,7 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.csv.CsvUtil;
import org.openmetadata.schema.api.data.BulkColumnUpdatePreview;
import org.openmetadata.schema.api.data.BulkColumnUpdateRequest;
import org.openmetadata.schema.api.data.ColumnGridResponse;
@ -356,7 +357,7 @@ public class ColumnResource {
@GET
@Path("/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({"text/csv; charset=UTF-8"})
@Operation(
operationId = "exportUniqueColumns",
summary = "Export unique column names to CSV",
@ -370,7 +371,7 @@ public class ColumnResource {
description = "CSV export of unique columns",
content =
@Content(
mediaType = "text/plain",
mediaType = "text/csv; charset=UTF-8",
schema = @Schema(implementation = String.class))),
@ApiResponse(responseCode = "400", description = "Bad request")
})
@ -396,13 +397,20 @@ public class ColumnResource {
String schemaName,
@Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId) {
return repository.exportUniqueColumnsCSV(
securityContext, columnName, entityTypes, serviceName, databaseName, schemaName, domainId);
return CsvUtil.withUtf8Bom(
repository.exportUniqueColumnsCSV(
securityContext,
columnName,
entityTypes,
serviceName,
databaseName,
schemaName,
domainId));
}
@POST
@Path("/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Operation(
operationId = "importUniqueColumns",
summary = "Import column metadata from CSV (with dry-run)",
@ -442,11 +450,12 @@ public class ColumnResource {
@Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId,
String csv) {
String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
CsvImportResult result =
repository.importColumnsCSV(
uriInfo,
securityContext,
csv,
normalizedCsv,
dryRun,
entityTypes,
serviceName,
@ -459,7 +468,7 @@ public class ColumnResource {
@POST
@Path("/import-async")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Operation(
operationId = "importUniqueColumnsAsync",
summary = "Import column metadata from CSV asynchronously",
@ -492,6 +501,7 @@ public class ColumnResource {
@Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId,
String csv) {
String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
String jobId = UUID.randomUUID().toString();
CSVImportResponse responseEntity =
new CSVImportResponse(jobId, "CSV column import is in progress.");
@ -508,7 +518,7 @@ public class ColumnResource {
repository.importColumnsCSV(
uriInfo,
securityContext,
csv,
normalizedCsv,
false,
entityTypes,
serviceName,

17
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
View file

@ -88,6 +88,7 @@ import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.FullyQualifiedName;
@Path("/v1/tables")
@ -582,19 +583,19 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@GET
@Path("/name/{name}/exportAsync")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportTable",
summary = "Export table in CSV format",
responses = {
@ApiResponse(
responseCode = "200",
description = "Exported csv with columns from the table",
responseCode = "202",
description = "Export initiated successfully",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public Response exportCsvAsync(
@Context SecurityContext securityContext,
@ -606,7 +607,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({"text/csv; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportTable",
@ -617,7 +618,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
description = "Exported csv with columns from the table",
content =
@Content(
mediaType = "application/json",
mediaType = "text/csv; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(
@ -631,7 +632,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@PUT
@Path("/name/{name}/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importTable",
@ -665,7 +666,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
@PUT
@Path("/name/{name}/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importTableAsync",

10
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
View file

@ -1252,7 +1252,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({"text/csv; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportTestCases",
@ -1267,7 +1267,9 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
responseCode = "200",
description = "Exported CSV with test cases",
content =
@Content(mediaType = "text/plain", schema = @Schema(implementation = String.class)))
@Content(
mediaType = "text/csv; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(
@Context SecurityContext securityContext,
@ -1315,7 +1317,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
@PUT
@Path("/name/{name}/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
@ -1360,7 +1362,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
@PUT
@Path("/name/{name}/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(

14
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java
View file

@ -536,15 +536,15 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
@GET
@Path("/name/{name}/exportAsync")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportGlossary",
summary = "Export glossary in CSV format",
responses = {
@ApiResponse(
responseCode = "200",
description = "Exported csv with glossary terms",
responseCode = "202",
description = "Export initiated successfully",
content =
@Content(
mediaType = "application/json",
@ -560,7 +560,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportGlossary",
@ -571,7 +571,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
description = "Exported csv with glossary terms",
content =
@Content(
mediaType = "application/json",
mediaType = "text/plain; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(
@ -585,7 +585,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
@PUT
@Path("/name/{name}/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importGlossary",
@ -619,7 +619,7 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
@PUT
@Path("/name/{name}/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(

16
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java
View file

@ -91,6 +91,7 @@ import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.security.policyevaluator.ResourceContextInterface;
import org.openmetadata.service.util.AsyncService;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.MoveGlossaryTermResponse;
@ -1324,7 +1325,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@GET
@Path("/name/{fqn}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({"text/csv; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportGlossaryTerm",
@ -1334,7 +1335,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@ApiResponse(
responseCode = "200",
description = "Exported csv with glossary terms",
content = @Content(mediaType = "text/plain"))
content = @Content(mediaType = "text/csv; charset=UTF-8"))
})
public String exportCsv(
@Context SecurityContext securityContext,
@ -1349,7 +1350,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@GET
@Path("/name/{fqn}/exportAsync")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportGlossaryTermAsync",
@ -1362,10 +1363,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
content =
@Content(
mediaType = "application/json",
schema =
@Schema(
implementation =
org.openmetadata.service.util.CSVExportResponse.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public Response exportCsvAsync(
@Context SecurityContext securityContext,
@ -1379,7 +1377,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@PUT
@Path("/name/{fqn}/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importGlossaryTerm",
@ -1417,7 +1415,7 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
@PUT
@Path("/name/{fqn}/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java
View file

@ -401,7 +401,7 @@ public class LineageResource {
@GET
@Path("/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({"text/csv; charset=UTF-8"})
@Operation(
operationId = "exportLineage",
summary = "Export lineage",
@ -411,8 +411,8 @@ public class LineageResource {
description = "search response",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = SearchResponse.class)))
mediaType = "text/csv; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportLineage(
@Context UriInfo uriInfo,

14
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java
View file

@ -719,15 +719,15 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
@GET
@Path("/name/{name}/exportAsync")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportTeams",
summary = "Export teams in CSV format",
responses = {
@ApiResponse(
responseCode = "200",
description = "Exported csv with teams information",
responseCode = "202",
description = "Export initiated successfully",
content =
@Content(
mediaType = "application/json",
@ -740,7 +740,7 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportTeams",
@ -751,7 +751,7 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
description = "Exported csv with teams information",
content =
@Content(
mediaType = "application/json",
mediaType = "text/plain; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name)
@ -761,7 +761,7 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
@PUT
@Path("/name/{name}/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importTeams",
@ -854,7 +854,7 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
@PUT
@Path("/name/{name}/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(

16
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View file

@ -1688,7 +1688,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
@GET
@Path("/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "exportUsers",
@ -1699,7 +1699,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
description = "Exported csv with user information",
content =
@Content(
mediaType = "application/json",
mediaType = "text/plain; charset=UTF-8",
schema = @Schema(implementation = String.class)))
})
public String exportUsersCsv(
@ -1716,11 +1716,11 @@ public class UserResource extends EntityResource<User, UserRepository> {
@PUT
@Path("/import")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importTeams",
summary = "Import from CSV to create, and update teams.",
operationId = "importUsers",
summary = "Import from CSV to create, and update users.",
responses = {
@ApiResponse(
responseCode = "200",
@ -1753,11 +1753,11 @@ public class UserResource extends EntityResource<User, UserRepository> {
@PUT
@Path("/importAsync")
@Consumes(MediaType.TEXT_PLAIN)
@Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
@Valid
@Operation(
operationId = "importTeamsAsync",
summary = "Import from CSV to create, and update teams asynchronously.",
operationId = "importUsersAsync",
summary = "Import from CSV to create, and update users asynchronously.",
responses = {
@ApiResponse(
responseCode = "200",

39
openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java
View file

@ -32,6 +32,8 @@ import org.openmetadata.schema.entity.type.CustomProperty;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TermRelation;
import org.openmetadata.schema.type.csv.CsvFile;
import org.openmetadata.schema.type.csv.CsvHeader;
public class CsvUtilTest {
@Test
@ -322,4 +324,41 @@ public class CsvUtilTest {
assertEquals(expectedCsvRecords.get(i), actualCsvRecords.get(i));
}
}
@Test
void testFormatCsvPreservesChineseCharacters() throws Exception {
CsvFile csvFile =
new CsvFile()
.withHeaders(
List.of(
new CsvHeader().withName("name"),
new CsvHeader().withName("description"),
new CsvHeader().withName("owner")))
.withRecords(List.of(List.of("中文表", "这是中文描述", "数据平台团队")));
String csv = CsvUtil.formatCsv(csvFile);
assertTrue(csv.contains("中文表"));
assertTrue(csv.contains("这是中文描述"));
assertTrue(csv.contains("数据平台团队"));
}
@Test
void testStripUtf8Bom() {
String csv = "name,description\n中文表,中文描述";
assertEquals(csv, CsvUtil.stripUtf8Bom(CsvUtil.UTF8_BOM + csv));
assertEquals(csv, CsvUtil.stripUtf8Bom(csv));
assertEquals("", CsvUtil.stripUtf8Bom(""));
assertNull(CsvUtil.stripUtf8Bom(null));
}
@Test
void testWithUtf8Bom() {
String csv = "name,description\nunicode-name,unicode-description";
assertEquals(CsvUtil.UTF8_BOM + csv, CsvUtil.withUtf8Bom(csv));
assertEquals(CsvUtil.UTF8_BOM + csv, CsvUtil.withUtf8Bom(CsvUtil.UTF8_BOM + csv));
assertEquals(CsvUtil.UTF8_BOM, CsvUtil.withUtf8Bom(""));
assertNull(CsvUtil.withUtf8Bom(null));
}
}

11
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
View file

@ -63,6 +63,13 @@ const propertiesList = Object.values(CUSTOM_PROPERTIES_TYPES);
const propertyListName: Record<string, string> = {};
const additionalGlossaryTerm = createGlossaryTermRowDetails();
const chineseGlossaryTermDetails = {
name: `术语${uuid()}`,
displayName: '中文术语展示名',
description: '这是用于验证导入导出编码的中文描述。',
synonyms: '中文同义词;测试',
references: '参考;https://example.com/%E4%B8%AD%E6%96%87',
};
test.describe('Glossary Bulk Import Export', () => {
test.slow(true);
@ -183,6 +190,7 @@ test.describe('Glossary Bulk Import Export', () => {
await fillGlossaryRowDetails(
{
...additionalGlossaryTerm,
...chineseGlossaryTermDetails,
owners: [user1.responseData?.['displayName']],
reviewers: [user2.responseData?.['displayName']],
relatedTerm: {
@ -214,6 +222,9 @@ test.describe('Glossary Bulk Import Export', () => {
const rowStatus = ['Entity updated', 'Entity created'];
await expect(page.locator('.rdg-cell-details')).toHaveText(rowStatus);
await expect(
page.getByText(chineseGlossaryTermDetails.name)
).toBeVisible();
await page.getByRole('button', { name: 'Update' }).click();
await page

2
openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.interface.ts
View file

@ -21,5 +21,5 @@ export interface UploadFileProps {
file: RcFile,
FileList: RcFile[]
) => BeforeUploadValueType | Promise<BeforeUploadValueType>;
onCSVUploaded: (event: ProgressEvent<FileReader>) => void;
onCSVUploaded: (event: ProgressEvent<FileReader>) => void | Promise<void>;
}

57
openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.test.tsx
View file

@ -34,7 +34,14 @@ describe('UploadFile Component', () => {
onCSVUploaded: jest.fn(),
};
let originalFileReader: typeof FileReader;
beforeEach(() => {
originalFileReader = global.FileReader;
});
afterEach(() => {
global.FileReader = originalFileReader;
jest.clearAllMocks();
});
@ -74,6 +81,16 @@ describe('UploadFile Component', () => {
it('should call onCSVUploaded when file is uploaded successfully', async () => {
const mockOnCSVUploaded = jest.fn();
const readAsText = jest.fn(function (this: FileReader) {
this.onload?.({ target: this } as ProgressEvent<FileReader>);
});
const MockFileReader = jest.fn().mockImplementation(() => ({
error: null,
onerror: null,
onload: null,
readAsText,
}));
global.FileReader = MockFileReader as unknown as typeof FileReader;
render(<UploadFile {...defaultProps} onCSVUploaded={mockOnCSVUploaded} />);
@ -91,6 +108,8 @@ describe('UploadFile Component', () => {
await waitFor(() => {
expect(mockOnCSVUploaded).toHaveBeenCalled();
});
expect(readAsText).toHaveBeenCalledWith(file, 'utf-8');
});
it('should handle file upload error', async () => {
@ -106,13 +125,12 @@ describe('UploadFile Component', () => {
});
// Mock FileReader to throw an error
const originalFileReader = global.FileReader;
global.FileReader = jest.fn().mockImplementation(() => ({
readAsText: jest.fn(() => {
throw new Error('File read error');
}),
onerror: null,
})) as any;
})) as unknown as typeof FileReader;
fireEvent.drop(uploadWidget, {
dataTransfer: {
@ -123,9 +141,40 @@ describe('UploadFile Component', () => {
await waitFor(() => {
expect(showErrorToast).toHaveBeenCalled();
});
});
// Restore original FileReader
global.FileReader = originalFileReader;
it('should show error toast when onCSVUploaded rejects', async () => {
const error = new Error('Upload failed');
const mockOnCSVUploaded = jest.fn().mockRejectedValue(error);
const readAsText = jest.fn(function (this: FileReader) {
this.onload?.({ target: this } as ProgressEvent<FileReader>);
});
const MockFileReader = jest.fn().mockImplementation(() => ({
error: null,
onerror: null,
onload: null,
readAsText,
}));
global.FileReader = MockFileReader as unknown as typeof FileReader;
render(<UploadFile {...defaultProps} onCSVUploaded={mockOnCSVUploaded} />);
const uploadWidget = screen.getByTestId('upload-file-widget');
const file = new File(['test,csv,content'], 'test.csv', {
type: 'text/csv',
});
fireEvent.drop(uploadWidget, {
dataTransfer: {
files: [file],
},
});
await waitFor(() => {
expect(showErrorToast).toHaveBeenCalledWith(error);
});
expect(readAsText).toHaveBeenCalledWith(file, 'utf-8');
});
it('should call beforeUpload when provided', () => {

25
openmetadata-ui/src/main/resources/ui/src/components/UploadFile/UploadFile.tsx
View file

@ -35,17 +35,22 @@ const UploadFile: FC<UploadFileProps> = ({
const handleUpload: UploadProps['customRequest'] = useCallback(
(options: UploadRequestOption) => {
setUploading(true);
const reader = new FileReader();
reader.onload = (e) => {
setUploading(false);
onCSVUploaded(e);
};
reader.onerror = () => {
setUploading(false);
showErrorToast(new Error(t('server.unexpected-error')) as AxiosError);
};
try {
reader.readAsText(options.file as Blob);
const reader = new FileReader();
reader.onload = async (event) => {
try {
await onCSVUploaded(event);
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setUploading(false);
}
};
reader.onerror = () => {
showErrorToast(reader.error?.message ?? t('server.unexpected-error'));
setUploading(false);
};
reader.readAsText(options.file as Blob, 'utf-8');
} catch (error) {
setUploading(false);
showErrorToast(error as AxiosError);

4
openmetadata-ui/src/main/resources/ui/src/rest/columnAPI.ts
View file

@ -166,7 +166,7 @@ export const importColumnsCSV = async (
`/columns/import?${queryParams.toString()}`,
params.csv,
{
headers: { 'Content-Type': 'text/plain' },
headers: { 'Content-Type': 'text/plain; charset=UTF-8' },
}
);
@ -198,7 +198,7 @@ export const importColumnsCSVAsync = async (
string,
AxiosResponse<CSVImportResponse>
>(`/columns/import-async?${queryParams.toString()}`, params.csv, {
headers: { 'Content-Type': 'text/plain' },
headers: { 'Content-Type': 'text/plain; charset=UTF-8' },
});
return response.data;

4
openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts
View file

@ -307,7 +307,7 @@ export const importDatabaseInCSVFormat = async (
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const res = await APIClient.put(
`/databases/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`,
@ -340,7 +340,7 @@ export const importDatabaseSchemaInCSVFormat = async (
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const res = await APIClient.put(
`/databaseSchemas/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`,

40
openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.test.ts
View file

@ -50,7 +50,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
expect(result).toEqual(mockCSVImportResponse);
});
@ -78,7 +78,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=false&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -105,7 +105,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/dataQuality/testCases/name/test.suite.name/importAsync?dryRun=true&recursive=true',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -132,7 +132,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
expect.stringContaining('/dataQuality/testCases/name/'),
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -177,7 +177,7 @@ describe('importExportAPI tests', () => {
});
expect(mockPut).toHaveBeenCalledWith(expect.any(String), '', {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
});
});
});
@ -205,7 +205,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/tables/name/database.schema.table/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
expect(result).toEqual(mockCSVImportResponse);
});
@ -232,7 +232,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/databases/name/service.database/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -258,7 +258,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/databaseSchemas/name/service.database.schema/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -286,7 +286,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/tables/name/database.schema.table/importAsync?dryRun=false&recursive=true',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -312,7 +312,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
expect.any(String),
expect.any(String),
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -361,7 +361,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'services/databaseServices/name/my-database-service/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
expect(result).toEqual(mockCSVImportResponse);
});
@ -388,7 +388,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'services/messagingServices/name/my-messaging-service/importAsync?dryRun=true&recursive=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -416,7 +416,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'services/databaseServices/name/my-service/importAsync?dryRun=false&recursive=true',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -465,7 +465,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/glossaries/name/my-glossary/importAsync?dryRun=true',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
expect(result).toEqual(mockCSVImportResponse);
});
@ -493,7 +493,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/glossaries/name/my-glossary/importAsync?dryRun=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -547,7 +547,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
expect.stringContaining('/glossaries/name/'),
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -596,7 +596,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/glossaryTerms/name/glossary.term/importAsync?dryRun=true',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
expect(result).toEqual(mockCSVImportResponse);
});
@ -624,7 +624,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
'/glossaryTerms/name/glossary.term/importAsync?dryRun=false',
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -678,7 +678,7 @@ describe('importExportAPI tests', () => {
expect(mockPut).toHaveBeenCalledWith(
expect.stringContaining('/glossaryTerms/name/'),
mockCSVData,
{ headers: { 'Content-type': 'text/plain' } }
{ headers: { 'Content-type': 'text/plain; charset=UTF-8' } }
);
});
@ -724,7 +724,7 @@ describe('importExportAPI tests', () => {
});
expect(mockPut).toHaveBeenCalledWith(expect.any(String), largeCsvData, {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
});
});
});

10
openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts
View file

@ -33,7 +33,7 @@ export const importTestCaseInCSVFormat = async ({
targetEntityType,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
let url = `/dataQuality/testCases/name/${getEncodedFqn(
name
@ -58,7 +58,7 @@ export const importEntityInCSVFormat = async ({
recursive = false,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const res = await APIClient.put<
string,
@ -82,7 +82,7 @@ export const importServiceInCSVFormat = async ({
recursive = false,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const res = await APIClient.put<
string,
@ -104,7 +104,7 @@ export const importGlossaryInCSVFormat = async ({
dryRun = true,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const response = await APIClient.put<
string,
@ -124,7 +124,7 @@ export const importGlossaryTermInCSVFormat = async ({
dryRun = true,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const response = await APIClient.put<
string,

2
openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts
View file

@ -264,7 +264,7 @@ export const importTableInCSVFormat = async (
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
};
const res = await APIClient.put(
`/tables/name/${getEncodedFqn(name)}/import?dryRun=${dryRun}`,

4
openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts
View file

@ -130,7 +130,7 @@ export const importTeam = async (
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
params: {
dryRun,
},
@ -149,7 +149,7 @@ export const importUserInTeam = async (
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
headers: { 'Content-type': 'text/plain; charset=UTF-8' },
params: {
team,
dryRun,

43
openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.test.tsx
View file

@ -29,6 +29,10 @@ jest.mock('../ToastUtils', () => ({
}));
describe('ExportUtils', () => {
afterEach(() => {
jest.restoreAllMocks();
});
describe('downloadFile', () => {
const mockLink = {
href: '',
@ -36,6 +40,9 @@ describe('ExportUtils', () => {
style: { visibility: '' },
click: jest.fn(),
};
const originalBlob = global.Blob;
const originalCreateObjectURL = global.URL.createObjectURL;
const originalRevokeObjectURL = global.URL.revokeObjectURL;
let mockCreateObjectURL: jest.Mock;
let mockRevokeObjectURL: jest.Mock;
@ -57,7 +64,9 @@ describe('ExportUtils', () => {
});
afterEach(() => {
jest.restoreAllMocks();
global.Blob = originalBlob;
global.URL.createObjectURL = originalCreateObjectURL;
global.URL.revokeObjectURL = originalRevokeObjectURL;
});
it('creates an anchor element and triggers a click', () => {
@ -92,15 +101,39 @@ describe('ExportUtils', () => {
expect(mockRevokeObjectURL).toHaveBeenCalledWith('blob:mock-url');
});
it('uses the provided mimeType when creating the Blob', () => {
it('normalizes CSV mimeType and prepends BOM when creating the Blob', () => {
const mockBlob = {};
const MockBlob = jest.fn().mockReturnValue(mockBlob);
global.Blob = MockBlob as unknown as typeof Blob;
downloadFile('content', 'file.csv', 'text/csv;charset=utf-8;');
expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], {
type: 'text/csv; charset=utf-8',
});
});
it('does not prepend a duplicate BOM when content already has one', () => {
const mockBlob = {};
const MockBlob = jest.fn().mockReturnValue(mockBlob);
global.Blob = MockBlob as unknown as typeof Blob;
downloadFile('\uFEFFcontent', 'file.csv', 'text/csv;charset=utf-8;');
expect(MockBlob).toHaveBeenCalledWith(['\uFEFFcontent'], {
type: 'text/csv; charset=utf-8',
});
});
it('does not prepend BOM for non-csv files', () => {
const mockBlob = {};
const MockBlob = jest.fn().mockReturnValue(mockBlob);
global.Blob = MockBlob as unknown as typeof Blob;
downloadFile('content', 'file.txt');
expect(MockBlob).toHaveBeenCalledWith(['content'], {
type: 'text/csv;charset=utf-8;',
type: 'text/plain',
});
});
});
@ -121,10 +154,6 @@ describe('ExportUtils', () => {
} as unknown as HTMLAnchorElement);
});
afterEach(() => {
mockCreateElement.mockRestore();
});
it('should create and trigger download with correct attributes', () => {
const dataUrl = 'data:image/png;base64,test';
const fileName = 'test-image';

12
openmetadata-ui/src/main/resources/ui/src/utils/Export/ExportUtils.ts
View file

@ -23,7 +23,17 @@ export const downloadFile = (
fileName: string,
mimeType: string = 'text/plain'
): void => {
const blob = new Blob([content], { type: mimeType });
const isCsvFile = fileName.toLowerCase().endsWith('.csv');
const isCsvMime = mimeType.toLowerCase().includes('text/csv');
const csvMimeType = 'text/csv; charset=utf-8';
const effectiveMimeType = isCsvFile || isCsvMime ? csvMimeType : mimeType;
const effectiveContent =
isCsvFile || isCsvMime
? content.startsWith('\uFEFF')
? content
: `\uFEFF${content}`
: content;
const blob = new Blob([effectiveContent], { type: effectiveMimeType });
const link = document.createElement('a');
link.href = URL.createObjectURL(blob);