Project

General

Profile

« Previous | Next » 

Revision 86ea4f4a

Added by Andreas Müller over 7 years ago

ref #3934 latest changes to Salvador import

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/app/berlinModelImport/SalvadorActivator.java
47 47
	static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
48 48
	static final Source berlinModelSource = BerlinModelSources.El_Salvador();
49 49
//	static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
50
    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_salvador_preview();
51
//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_salvador_production();
50
//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_salvador_preview();
51
    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_salvador_production();
52 52
	static final UUID treeUuid = UUID.fromString("b010c84d-6049-45f4-9f13-c065101eaa26");
53 53
	static final UUID secUuid = UUID.fromString("d03ef02a-f226-4cb1-bdb4-f6c154f08a34");
54 54
	static final int sourceSecId = 7331;
55 55

  
56 56
	static final UUID featureTreeUuid = UUID.fromString("9d0e5628-2eda-43ed-bc59-138a7e39ce56");
57
	static final Object[] featureKeyList = new Integer[]{302, 303, 306, 307, 309, 310, 311, 312, 350, 1500, 1800, 1900, 1950, 1980, 2000, 10299};
57
	static final Object[] featureKeyList = new Integer[]{302, 303, 306, 307, 309, 310, 311, 312, 350, 1500, 1800, 1900, 1950, 1980, 2000};
58 58
	static boolean isIgnore0AuthorTeam = true;  //special case for Salvador.
59 59
	static boolean doExport = false;
60 60
	static boolean useClassification = true;
......
96 96
	static final boolean doUser = true;
97 97

  
98 98

  
99
	static String factFilter = " factCategoryFk NOT IN  (1980, 1500, 1950, 1700) ";
99
	static String factFilter = " factCategoryFk NOT IN  ("
100
//	        + "302, 303, 306, 307, 309, 311, 310, "
101
	        + "1980, 1500, 1950, 1700) ";
100 102

  
101 103

  
102 104
// ************************ NONE **************************************** //
......
154 156
		config.setDoMarker(doMarker);
155 157
		config.setDoUser(doUser);
156 158

  
159
		config.setTaxonNoteAsFeature(true);
160

  
157 161
		config.setDbSchemaValidation(hbm2dll);
158 162

  
159 163
		config.setCheck(check);
......
166 170
		config.setSalvador(isSalvador);
167 171
		config.setFactFilter(factFilter);
168 172

  
173
		config.setFeatureTreeUuid(featureTreeUuid);
174
		config.setFeatureTreeTitle("Salvador Portal Feature Tree");
175

  
169 176
		// invoke import
170 177
		CdmDefaultImport<BerlinModelImportConfigurator> bmImport = new CdmDefaultImport<BerlinModelImportConfigurator>();
171 178
		ImportResult result = bmImport.invoke(config);
......
178 185

  
179 186
    //create feature tree
180 187
    private void createFeatureTree(BerlinModelImportConfigurator config,
181
            CdmDefaultImport<BerlinModelImportConfigurator> bmImport)
182
    {
188
            CdmDefaultImport<BerlinModelImportConfigurator> bmImport){
183 189
        if (config.isDoFacts() && (config.getCheck().isImport()  )  ){
184 190
            try {
185 191
                ICdmApplicationConfiguration app = bmImport.getCdmAppController();
186
                TransactionStatus tx = app.startTransaction();
192
                TransactionStatus tx = app.startTransaction(false);
187 193

  
188 194
                //make feature tree
189 195
                FeatureTree tree = TreeCreator.flatTree(featureTreeUuid, config.getFeatureMap(), featureKeyList);
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelFactsImport.java
58 58
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
59 59
import eu.etaxonomy.cdm.model.description.Distribution;
60 60
import eu.etaxonomy.cdm.model.description.Feature;
61
import eu.etaxonomy.cdm.model.description.FeatureNode;
62
import eu.etaxonomy.cdm.model.description.FeatureTree;
61 63
import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;
62 64
import eu.etaxonomy.cdm.model.description.TaxonDescription;
63 65
import eu.etaxonomy.cdm.model.description.TextData;
64
import eu.etaxonomy.cdm.model.location.Country;
65 66
import eu.etaxonomy.cdm.model.location.NamedArea;
66 67
import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
67 68
import eu.etaxonomy.cdm.model.location.NamedAreaType;
......
124 125

  
125 126
		Map<Integer, Feature>  result = state.getConfig().getFeatureMap();
126 127
		Source source = state.getConfig().getSource();
128
        boolean createFeatureTree = state.getConfig().isSalvador();  //for some reason feature tree creation does not work for salavdor
127 129

  
128
		try {
130
        FeatureTree featureTree = (!createFeatureTree) ? null : FeatureTree.NewInstance(state.getConfig().getFeatureTreeUuid());
131
        if (createFeatureTree){
132
            featureTree.setTitleCache(state.getConfig().getFeatureTreeTitle(), true);
133
        }
134

  
135
        try {
129 136
			//get data from database
130 137
			String strQuery =
131 138
					" SELECT FactCategory.* " +
......
173 180
				}
174 181

  
175 182
				result.put(factCategoryId, feature);
183
				if (createFeatureTree){
184
				    featureTree.getRoot().addChild(FeatureNode.NewInstance(feature));
185
				}
186
			}
187
			if (createFeatureTree){
188
			    getFeatureTreeService().save(featureTree);
176 189
			}
177 190
			return result;
178 191
		} catch (SQLException e) {
......
388 401

  
389 402
						if (state.getConfig().isSalvador()){
390 403
						    if (categoryFkInt == 306){
391
						        deb = CommonTaxonName.NewInstance(fact, Language.SPANISH_CASTILIAN(), Country.ELSALVADORREPUBLICOF());
404
						        NamedArea area = null;  // for now we do not set an area as it can not be disabled in dataportals via css yet
405
						        deb = CommonTaxonName.NewInstance(fact, Language.SPANISH_CASTILIAN(), area);
392 406
						    }else if (categoryFkInt == 307){
393 407
						        Distribution salvadorDistribution = salvadorDistributionFromMuestrasDeHerbar((Taxon)taxonBase, fact);
394 408
						        if (salvadorDistribution != null){
......
914 928
        vol2.setInSeries(englera);
915 929
        vol3.setInSeries(englera);
916 930

  
917
        vol1.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 1: Angiospermae - Familias A-L");
918
        vol2.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 2: Angiospermae - Familias M-P y Pteridophyta");
919
        vol3.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 3: Angiospermae - Familias R-Z y Gymnospermae");
931
        vol1.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 1: Angiospermae - Familias A a L");
932
        vol2.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 2: Angiospermae - Familias M a P y Pteridophyta");
933
        vol3.setTitle("Nova Silva Cuscatlanica, Árboles nativos e introducidos de El Salvador - Parte 3: Angiospermae - Familias R a Z y Gymnospermae");
920 934

  
921 935
        vol1.setVolume("29(1)");
922 936
        vol2.setVolume("29(2)");
......
926 940
        vol2.setVolume("1-300");
927 941
        vol3.setVolume("1-356");
928 942

  
943
        String placePublished = "Berlin: Botanic Garden and Botanical Museum Berlin; Antiguo Cuscatlán: Asociación Jardín Botánico La Laguna, El Salvador";
944
        vol1.setPlacePublished(placePublished);
945
        vol2.setPlacePublished(placePublished);
946
        vol3.setPlacePublished(placePublished);
947

  
929 948
        salvadorRef1Id = getReferenceService().save(vol1).getId();
930 949
        salvadorRef2Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol2)).getId();
931 950
        salvadorRef3Id = getReferenceService().find(getReferenceService().saveOrUpdate(vol3)).getId();
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelImportConfigurator.java
13 13
import java.lang.reflect.Method;
14 14
import java.net.MalformedURLException;
15 15
import java.net.URL;
16
import java.util.UUID;
16 17

  
17 18
import org.apache.log4j.Logger;
18 19

  
......
20 21
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport.PublishMarkerChooser;
21 22
import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelGeneralImportValidator;
22 23
import eu.etaxonomy.cdm.io.common.DbImportConfiguratorBase;
23
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
24 24
import eu.etaxonomy.cdm.io.common.Source;
25 25
import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
26 26
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
......
29 29
 * @author a.mueller
30 30
 * @created 20.03.2008
31 31
 */
32
public class BerlinModelImportConfigurator extends DbImportConfiguratorBase<BerlinModelImportState> implements IImportConfigurator{
32
public class BerlinModelImportConfigurator extends DbImportConfiguratorBase<BerlinModelImportState>{
33 33
    private static final long serialVersionUID = 70300913255425256L;
34 34

  
35 35
    private static Logger logger = Logger.getLogger(BerlinModelImportConfigurator.class);
......
110 110
	//specific functions
111 111
	private Method 	makeUrlForTaxon = null;
112 112

  
113
	@Override
113
	private UUID featureTreeUuid;
114
	private String featureTreeTitle;
115

  
116
    private boolean isTaxonNoteAsFeature = false;
117

  
118

  
119
    public boolean isTaxonNoteAsFeature() {return isTaxonNoteAsFeature;}
120
    public void setTaxonNoteAsFeature(boolean isTaxonNoteAsFeature) {this.isTaxonNoteAsFeature = isTaxonNoteAsFeature;}
121

  
122
    @Override
114 123
    protected void makeIoClassList(){
115 124
		ioClassList = new Class[]{
116 125
				BerlinModelGeneralImportValidator.class
......
632 641
    public boolean isSalvador() {return isSalvador;}
633 642
    public void setSalvador(boolean isSalvador) {this.isSalvador = isSalvador;}
634 643

  
644
    public UUID getFeatureTreeUuid() {
645
        return featureTreeUuid;
646
    }
647
    public void setFeatureTreeUuid(UUID featureTreeUuid) {
648
        this.featureTreeUuid = featureTreeUuid;
649
    }
650

  
651
    public String getFeatureTreeTitle() {
652
        return featureTreeTitle;
653
    }
654
    public void setFeatureTreeTitle(String featureTreeTitle) {
655
        this.featureTreeTitle = featureTreeTitle;
656
    }
657

  
658

  
659

  
660

  
635 661
}
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java
76 76
 */
77 77
@Component
78 78
public class BerlinModelReferenceImport extends BerlinModelImportBase {
79
	private static final Logger logger = Logger.getLogger(BerlinModelReferenceImport.class);
79
    private static final long serialVersionUID = -3667566958769967591L;
80

  
81
    private static final Logger logger = Logger.getLogger(BerlinModelReferenceImport.class);
80 82

  
81 83
	public static final String REFERENCE_NAMESPACE = "Reference";
84
	private static final String REF_AUTHOR_NAMESPACE = "Reference.refAuthorString";
85

  
82 86

  
83 87
	public static final UUID REF_DEPOSITED_AT_UUID = UUID.fromString("23ca88c7-ce73-41b2-8ca3-2cb22f013beb");
84 88
	public static final UUID REF_SOURCE_UUID = UUID.fromString("d6432582-2216-4b08-b0db-76f6c1013141");
......
239 243

  
240 244
		Map<Integer, Reference> refToSave = new HashMap<Integer, Reference>();
241 245

  
242
		Map<String, Reference> relatedReferences = partitioner.getObjectMap(REFERENCE_NAMESPACE);
246
		@SuppressWarnings("unchecked")
247
        Map<String, Reference> relatedReferences = partitioner.getObjectMap(REFERENCE_NAMESPACE);
243 248

  
244 249
		BerlinModelImportConfigurator config = state.getConfig();
245 250

  
......
350 355
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
351 356

  
352 357
		try{
353
			Set<String> teamIdSet = new HashSet<String>();
354
			Set<String> referenceIdSet = new HashSet<String>();
358
			Set<String> teamIdSet = new HashSet<>();
359
			Set<String> referenceIdSet = new HashSet<>();
360
			Set<String> teamStringSet = new HashSet<>();
355 361

  
356 362
			while (rs.next()){
357 363
				handleForeignKey(rs, teamIdSet, "NomAuthorTeamFk");
358 364
				handleForeignKey(rs, referenceIdSet, "InRefFk");
365
				handleForeignKey(rs, teamStringSet, "refAuthorString");
359 366
				//TODO only needed in second path but state not available here to check if state is second path
360 367
				handleForeignKey(rs, referenceIdSet, "refId");
361 368
			}
362 369

  
370
			Set<String> teamStringSet2 = new HashSet<>();
371
			for (String teamString : teamStringSet){
372
			    teamStringSet2.add(teamString.replace("'", "´"));
373
			}
374

  
363 375
			//team map
364 376
			nameSpace = BerlinModelAuthorTeamImport.NAMESPACE;
365 377
			cdmClass = Team.class;
366 378
			idSet = teamIdSet;
367
			Map<String, Team> teamMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
379
			@SuppressWarnings("unchecked")
380
            Map<String, Team> teamMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
368 381
			result.put(nameSpace, teamMap);
369 382

  
383
            //refAuthor map
384
            nameSpace = REF_AUTHOR_NAMESPACE;
385
            cdmClass = Team.class;
386
            idSet = teamStringSet2;
387
            @SuppressWarnings("unchecked")
388
            Map<String, Team> refAuthorMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
389
            result.put(nameSpace, refAuthorMap);
390

  
370 391
			//reference map
371 392
			nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
372 393
			cdmClass = Reference.class;
......
399 420
				Map<Integer, Reference> refToSave,
400 421
				Map<String, Reference> relatedReferences,
401 422
				RefCounter refCounter){
402
		boolean success = true;
423

  
424
	    boolean success = true;
403 425

  
404 426
		Integer refId = null;
405 427
		try {
......
494 516
				Map<Integer, Reference> refToSave
495 517
				) throws SQLException{
496 518

  
497
		Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
519
		@SuppressWarnings("unchecked")
520
        Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
498 521

  
499 522
		String refCache = rs.getString("refCache");
500 523
		String nomRefCache = rs.getString("nomRefCache");
......
523 546
		}
524 547

  
525 548
		//author
526
		TeamOrPersonBase<?> author = getAuthorship(refAuthorString , nomAuthor);
549
		TeamOrPersonBase<?> author = getAuthorship(state, refAuthorString , nomAuthor);
527 550
		ref.setAuthorship(author);
528 551

  
529 552
		//save
......
694 717

  
695 718
		//Set bookAttributes = new String[]{"edition", "isbn", "pages","publicationTown","publisher","volume"};
696 719

  
697
		Set<String> omitAttributes = new HashSet<String>();
720
		Set<String> omitAttributes = new HashSet<>();
698 721
		String attrSeries = "series";
699 722
//		omitAttributes.add(attrSeries);
700 723

  
......
844 867
	}
845 868

  
846 869

  
847
	private static TeamOrPersonBase<?> getAuthorship(String authorString, TeamOrPersonBase<?> nomAuthor){
848
		TeamOrPersonBase<?> result;
870
	private static TeamOrPersonBase<?> getAuthorship(BerlinModelImportState state, String authorString, TeamOrPersonBase<?> nomAuthor){
871

  
872
	    TeamOrPersonBase<?> result;
849 873
		if (nomAuthor != null){
850 874
			result = nomAuthor;
851 875
		} else if (StringUtils.isNotBlank(authorString)){
852
			//FIXME check for existing team / persons
853
			TeamOrPersonBase<?> team = Team.NewInstance();
854
			team.setNomenclaturalTitle(authorString);
855
			team.setTitleCache(authorString, true);
856
			team.setNomenclaturalTitle(authorString);
876
			//TODO match with existing Persons/Teams
877
		    Team team = state.getRelatedObject(REF_AUTHOR_NAMESPACE, authorString, Team.class);
878
			if (team == null){
879
			    team = Team.NewInstance();
880
			    team.setNomenclaturalTitle(authorString);
881
			    team.setTitleCache(authorString, true);
882
			    state.addRelatedObject(REF_AUTHOR_NAMESPACE, authorString, team);
883
			    team.addImportSource(authorString, REF_AUTHOR_NAMESPACE, state.getConfig().getSourceReference(), null);
884
			}
857 885
			result = team;
858 886
		}else{
859 887
			result = null;
......
869 897
	 * @return
870 898
	 */
871 899
	public Set<String> getObligatoryAttributes(boolean lowerCase, BerlinModelImportConfigurator config){
872
		Set<String> result = new HashSet<String>();
900
		Set<String> result = new HashSet<>();
873 901
		Class<ICdmImport>[] ioClassList = config.getIoClassList();
874 902
		logger.warn("getObligatoryAttributes has been commented because it still needs to be adapted to the new package structure");
875 903
		result.addAll(Arrays.asList(unclearMappers));
......
881 909
		}
882 910
		result.addAll(mapping.getSourceAttributes());
883 911
		if (lowerCase){
884
			Set<String> lowerCaseResult = new HashSet<String>();
912
			Set<String> lowerCaseResult = new HashSet<>();
885 913
			for (String str : result){
886 914
				if (str != null){lowerCaseResult.add(str.toLowerCase());}
887 915
			}
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java
37 37
import eu.etaxonomy.cdm.model.common.CdmBase;
38 38
import eu.etaxonomy.cdm.model.common.Extension;
39 39
import eu.etaxonomy.cdm.model.common.ExtensionType;
40
import eu.etaxonomy.cdm.model.common.Language;
41
import eu.etaxonomy.cdm.model.description.TaxonDescription;
42
import eu.etaxonomy.cdm.model.description.TextData;
40 43
import eu.etaxonomy.cdm.model.name.TaxonNameBase;
41 44
import eu.etaxonomy.cdm.model.reference.Reference;
42 45
import eu.etaxonomy.cdm.model.taxon.Synonym;
......
47 50
/**
48 51
 * @author a.mueller
49 52
 * @created 20.03.2008
50
 * @version 1.0
51 53
 */
52 54
@Component
53 55
public class BerlinModelTaxonImport  extends BerlinModelImportBase {
54
	private static final Logger logger = Logger.getLogger(BerlinModelTaxonImport.class);
56
    private static final long serialVersionUID = -1186364983750790695L;
57

  
58
    private static final Logger logger = Logger.getLogger(BerlinModelTaxonImport.class);
55 59

  
56 60
	public static final String NAMESPACE = "Taxon";
57 61

  
......
255 259
					}
256 260

  
257 261
					//Notes
258
					doIdCreatedUpdatedNotes(state, taxonBase, rs, taxonId, NAMESPACE);
262
					boolean excludeNotes = state.getConfig().isTaxonNoteAsFeature() && taxonBase.isInstanceOf(Taxon.class);
263
					doIdCreatedUpdatedNotes(state, taxonBase, rs, taxonId, NAMESPACE, false, excludeNotes);
264
					if (excludeNotes){
265
					    makeTaxonomicNote(state, CdmBase.deproxy(taxonBase, Taxon.class), rs.getString("Notes"));
266
					}
259 267

  
260 268
					//external url
261 269
					if (config.getMakeUrlForTaxon() != null){
......
284 292
		return success;
285 293
	}
286 294

  
287
	@Override
295
	/**
296
     * @param state
297
     * @param taxonBase
298
	 * @param notes
299
     */
300
    private void makeTaxonomicNote(BerlinModelImportState state, Taxon taxon, String notes) {
301
        TaxonDescription desc = getTaxonDescription(taxon, false, true);
302
        desc.addElement(TextData.NewInstance(notes, Language.SPANISH_CASTILIAN(), null));
303
    }
304

  
305
    @Override
288 306
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
289 307
		String nameSpace;
290 308
		Class<?> cdmClass;
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java
300 300
							}
301 301
							handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
302 302
							Synonym synonym = (Synonym)taxon1;
303
							if (synonym.getAcceptedTaxon()!= null){
304
							    logger.warn("Synonym already has an accepted taxon. Create clone.");
305
							    synonym = (Synonym)synonym.clone();
306
							}
303 307
							makeSynRel(relQualifierFk, toTaxon, synonym, citation, microcitation);
304 308
							if (isNotBlank(notes)){
305 309
	                            logger.warn("Notes for synonym relationship or unknown taxon relationship not handled");

Also available in: Unified diff