Project

General

Profile

« Previous | Next » 

Revision 3a676b7a

Added by Andreas Müller over 6 years ago

ref #6606 changes in cdmlib for bogota specimen import

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java
1443 1443
	}
1444 1444

  
1445 1445

  
1446
    /**
1447
     * Converts a given string into an integer. If this is not possible
1448
     * an error is logged in the import result with record location and attribute name.
1449
     *
1450
     * @param state
1451
     * @param strToConvert
1452
     * @param recordLocation
1453
     * @param attributeName
1454
     * @return the converted integer
1455
     */
1456
    protected Integer intFromString(STATE state,
1457
            String strToConvert,
1458
            String recordLocation,
1459
            String attributeName) {
1460

  
1461
        if (strToConvert == null){
1462
            return null;
1463
        }
1464
        try {
1465
            Integer result = Integer.valueOf(strToConvert);
1466
            return result;
1467
        } catch (NumberFormatException e) {
1468
            String message = "Text '%s' could not be transformed into integer number for attribute %s";
1469
            message = String.format(message, strToConvert, attributeName);
1470
            state.getResult().addError(message, e, null, recordLocation);
1471
        }
1472
        return null;
1473
    }
1474

  
1475
    /**
1476
     * Converts a given string into a {@link Double}. If this is not possible
1477
     * an error is logged in the import result with record location and attribute name.
1478
     *
1479
     * @param state
1480
     * @param strToConvert
1481
     * @param recordLocation
1482
     * @param attributeName
1483
     * @return the converted integer
1484
     */
1485
    protected Double doubleFromString(STATE state,
1486
            String strToConvert,
1487
            String recordLocation,
1488
            String attributeName) {
1489

  
1490
        if (strToConvert == null){
1491
            return null;
1492
        }
1493
        try {
1494
            Double result = Double.valueOf(strToConvert);
1495
            return result;
1496
        } catch (NumberFormatException e) {
1497
            String message = "Text '%s' could not be transformed into number of type Double for attribute %s";
1498
            message = String.format(message, strToConvert, attributeName);
1499
            state.getResult().addError(message, e, null, recordLocation);
1500
        }
1501
        return null;
1502
    }
1503

  
1504

  
1446 1505
	/**
1447 1506
	 * Returns <code>null</code> for all blank strings. Identity function otherwise.
1448 1507
	 * @param str
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/utils/ImportDeduplicationHelper.java
21 21

  
22 22
import eu.etaxonomy.cdm.api.application.ICdmRepository;
23 23
import eu.etaxonomy.cdm.api.service.IService;
24
import eu.etaxonomy.cdm.common.CdmUtils;
24 25
import eu.etaxonomy.cdm.io.common.ImportResult;
25 26
import eu.etaxonomy.cdm.io.common.ImportStateBase;
26 27
import eu.etaxonomy.cdm.model.agent.AgentBase;
......
35 36
import eu.etaxonomy.cdm.model.name.HybridRelationship;
36 37
import eu.etaxonomy.cdm.model.name.INonViralName;
37 38
import eu.etaxonomy.cdm.model.name.TaxonName;
39
import eu.etaxonomy.cdm.model.occurrence.Collection;
38 40
import eu.etaxonomy.cdm.model.reference.INomenclaturalReference;
39 41
import eu.etaxonomy.cdm.model.reference.Reference;
40 42
import eu.etaxonomy.cdm.strategy.match.DefaultMatchStrategy;
......
57 59
    boolean nameMapIsInitialized = false;
58 60
    boolean agentMapIsInitialized = false;
59 61
    boolean copyrightMapIsInitialized = false;
62
    boolean collectionMapIsInitialized = false;
63

  
60 64

  
61 65
    private Map<String, Set<Reference>> refMap = new HashMap<>();
62
    private Map<String, Team> teamMap = new HashMap<>();
63
    private Map<String, Person> personMap = new HashMap<>();
66
    private Map<String, Set<Team>> teamMap = new HashMap<>();
67
    private Map<String, Set<Person>> personMap = new HashMap<>();
64 68
    private Map<String, Institution> institutionMap = new HashMap<>();
65 69
    //using titleCache
66 70
    private Map<String, Set<INonViralName>> nameMap = new HashMap<>();
67 71
    private Map<String, Set<Rights>> copyrightMap = new HashMap<>();
72
    private Map<String, Set<Collection>> collectionMap = new HashMap<>();
68 73

  
69 74

  
70 75
    private IMatchStrategy referenceMatcher = DefaultMatchStrategy.NewInstance(Reference.class);
76
//    private IMatchStrategy collectionMatcher = DefaultMatchStrategy.NewInstance(Collection.class);
71 77
    private IMatchStrategy nameMatcher = DefaultMatchStrategy.NewInstance(TaxonName.class);
78
    private IMatchStrategy personMatcher = DefaultMatchStrategy.NewInstance(Person.class);
79
    private IMatchStrategy teamMatcher = DefaultMatchStrategy.NewInstance(Team.class);
80

  
72 81

  
73 82

  
74 83

  
......
80 89
        if (repository == null){
81 90
            return;
82 91
        }
83
        personMap = refreshMap(personMap, (IService)repository.getAgentService(), importResult);
84
        teamMap = refreshMap(teamMap, (IService)repository.getAgentService(), importResult);
92
        refMap = refreshSetMap(refMap, (IService)repository.getReferenceService(), importResult);
93
        personMap = refreshSetMap(personMap, (IService)repository.getAgentService(), importResult);
94
        teamMap = refreshSetMap(teamMap, (IService)repository.getAgentService(), importResult);
85 95
        institutionMap = refreshMap(institutionMap, (IService)repository.getAgentService(), importResult);
96

  
97
        nameMap = refreshSetMap(nameMap, (IService)repository.getNameService(), importResult);
98
        collectionMap = refreshSetMap(collectionMap, (IService)repository.getCollectionService(), importResult);
99
        //TODO copyright ?
86 100
    }
87 101

  
88 102

  
......
112 126
        return newMap;
113 127
    }
114 128

  
129
    private <T extends ICdmBase> Map<String, Set<T>> refreshSetMap(Map<String, Set<T>> oldMap,
130
            IService<T> service, ImportResult importResult) {
131
        Map<String, Set<T>> newMap = new HashMap<>();
132
        for (String key : oldMap.keySet()){
133
            Set<T> oldSet = oldMap.get(key);
134
            Set<T> newSet = new HashSet<>();
135
            if (oldSet != null){
136
                newMap.put(key, newSet);
137
                for (T item : oldSet){
138
                    T cdmBase = service.find(item.getUuid());
139
                    if (cdmBase == null){
140
                        String message = "No cdm object was found for uuid " + item.getUuid() + " of class " + item.getClass().getSimpleName();
141
                        importResult.addWarning(message);
142
                    }else{
143
                        newSet.add(cdmBase);
144
                    }
145
                }
146
            }else{
147
                String message = "Value for key " +  key + " was null in deduplication map";
148
                importResult.addWarning(message);
149
            }
150
        }
151
        return newMap;
152
    }
153

  
115 154
// ************************** FACTORY *******************************/
116 155

  
117 156
    public static ImportDeduplicationHelper<?> NewInstance(ICdmRepository repository){
......
173 212
    // AGENTS
174 213
    private void putAgentBase(String title, AgentBase<?> agent){
175 214
        if (agent.isInstanceOf(Person.class) ){
176
            personMap.put(title, CdmBase.deproxy(agent, Person.class));
215
            putAgent(title, CdmBase.deproxy(agent, Person.class), personMap);
177 216
        }else if (agent.isInstanceOf(Team.class)){
178
            teamMap.put(title, CdmBase.deproxy(agent, Team.class));
217
            putAgent(title, CdmBase.deproxy(agent, Team.class), teamMap);
179 218
        }else{
219
//            putAgent(title, CdmBase.deproxy(agent, Institution.class), institutionMap);
180 220
            institutionMap.put(title, CdmBase.deproxy(agent, Institution.class));
181 221
        }
182 222
    }
183

  
184
    private TeamOrPersonBase<?> getAgentBase(String title){
185
        TeamOrPersonBase<?> result = personMap.get(title);
186
        if (result == null){
187
            result = teamMap.get(title);
223
    //put agent
224
    private <T extends AgentBase> void putAgent(String title, T agent, Map<String, Set<T>> map){
225
        Set<T> items = map.get(title);
226
        if (items == null){
227
            items = new HashSet<>();
228
            map.put(title, items);
229
        }
230
        items.add(agent);
231
    }
232
//
233
//    private TeamOrPersonBase<?> getTeamOrPerson(TeamOrPersonBase<?> agent){
234
//        TeamOrPersonBase<?> result = getMatchingPerson(agent) ; // personMap.get(title);
235
//        if (result == null){
236
//            result = teamMap.get(title);
237
//        }
238
//        return result;
239
//    }
240

  
241
    private Optional<Person> getMatchingPerson(Person existing){
242
        Predicate<Person> matchFilter = person ->{
243
            try {
244
                return personMatcher.invoke(person, existing);
245
            } catch (MatchException e) {
246
                throw new RuntimeException(e);
247
            }
248
        };
249
        return Optional.ofNullable(getPersons(existing.getTitleCache()))
250
                .orElse(new HashSet<>())
251
                .stream()
252
                .filter(matchFilter)
253
                .findAny();
254
    }
255
    private TeamOrPersonBase<?> getTeamOrPerson(TeamOrPersonBase<?> agent){
256
        TeamOrPersonBase<?> result = agent;
257
        if (agent.isInstanceOf(Person.class)){
258
            result = getMatchingPerson(CdmBase.deproxy(agent, Person.class)).orElse(null) ; // personMap.get(title);
259
        }else if (agent.isInstanceOf(Team.class)) {
260
            result = getMatchingTeam(CdmBase.deproxy(agent, Team.class)).orElse(null); // teamMap.get(title);
188 261
        }
189 262
        return result;
190 263
    }
191 264

  
192
    private Person getPerson(String title){
265
    private Optional<Team> getMatchingTeam(Team existing){
266
        Predicate<Team> matchFilter = person ->{
267
            try {
268
                return teamMatcher.invoke(person, existing);
269
            } catch (MatchException e) {
270
                throw new RuntimeException(e);
271
            }
272
        };
273
        return Optional.ofNullable(getTeam(existing.getTitleCache()))
274
                .orElse(new HashSet<>())
275
                .stream()
276
                .filter(matchFilter)
277
                .findAny();
278
    }
279
    private Set<Person> getPersons(String title){
193 280
        return personMap.get(title);
194 281
    }
282
    private Set<Team> getTeam(String title){
283
        return teamMap.get(title);
284
    }
195 285

  
196 286
    //NAMES
197 287
    private void putName(String title, INonViralName name){
......
221 311
                .findAny();
222 312
    }
223 313

  
314
    //COLLECTIONS
315
    private void putCollection(String title, Collection collection){
316
        Set<Collection> collections = collectionMap.get(title);
317
        if (collections == null){
318
            collections = new HashSet<>();
319
            collectionMap.put(title, collections);
320
        }
321
        collections.add(collection);
322
    }
323

  
324
    private Set<Collection> getCollections(String title){
325
        return collectionMap.get(title);
326
    }
327

  
328
    private Optional<Collection> getMatchingCollections(Collection existing){
329
        Predicate<Collection> matchFilter = collection ->{
330
//            try {
331
                //TODO right Collection matching
332
                if (CdmUtils.nullSafeEqual(collection.getName(), existing.getName())
333
                        && CdmUtils.nullSafeEqual(collection.getCode(), existing.getCode())){
334
                    return true;
335
                }else{
336
                    return false;
337
                }
338
//                return collectionMatcher.invoke(collection, existing);
339
//            } catch (MatchException e) {
340
//                throw new RuntimeException(e);
341
//            }
342
        };
343
        return Optional.ofNullable(getCollections(existing.getTitleCache()))
344
                .orElse(new HashSet<>())
345
                .stream()
346
                .filter(matchFilter)
347
                .findAny();
348
    }
349

  
224 350
// **************************** METHODS *****************************/
225 351

  
226 352
    /**
......
269 395
            return null;
270 396
        }else{
271 397
            initAgentMap(state);
272
            TeamOrPersonBase<?> result = getAgentBase(author.getTitleCache());
398
            TeamOrPersonBase<?> result = getTeamOrPerson(author);
273 399
            if (result == null){
274 400
                putAgentBase(author.getTitleCache(), author);
275 401
                if (author instanceof Team){
......
323 449
        List<Person> members = team.getTeamMembers();
324 450
        for (int i =0; i< members.size(); i++){
325 451
            Person person = members.get(i);
326
            Person existingPerson = getPerson(person.getTitleCache());
452
            Person existingPerson = getMatchingPerson(person).orElse(null);
327 453
            if (existingPerson != null){
328 454
                members.set(i, existingPerson);
329 455
            }else{
......
332 458
        }
333 459
    }
334 460

  
461

  
462
    /**
463
     * @param state
464
     * @param collection
465
     * @return
466
     */
467
    public Collection getExistingCollection(STATE state, Collection collection) {
468
        if (collection == null){
469
            return null;
470
        }else{
471
            initCollectionMap(state);
472
            Collection result = getMatchingCollections(collection).orElse(null);
473
            if (result == null){
474
                result = collection;
475
                putCollection(result.getTitleCache(), result);
476
            }else{
477
                if(logger.isDebugEnabled()) {
478
                    logger.debug("Matches");
479
                 }
480
            }
481
            return result;
482
        }
483
    }
484

  
485
    /**
486
     * @param state
487
     */
488
    private void initCollectionMap(STATE state) {
489
        if (!collectionMapIsInitialized && repository != null){
490
            List<String> propertyPaths = Arrays.asList("");
491
            List<Collection> existingCollections = repository.getCollectionService().list(null, null, null, null, propertyPaths);
492
            for (Collection collection : existingCollections){
493
                putCollection(collection.getTitleCache(), collection);
494
            }
495
            collectionMapIsInitialized = true;
496
        }
497
    }
498

  
335 499
    /**
336 500
    * @param state
337 501
    * @param nomRef
......
416 580
       }
417 581
   }
418 582

  
583

  
584

  
419 585
   public Rights getExistingCopyright(STATE state,
420 586
           Rights right) {
421 587
       if (right == null || !RightsType.COPYRIGHT().equals(right.getType())){
......
482 648
    }
483 649

  
484 650

  
651

  
485 652
}
cdmlib-model/src/main/java/eu/etaxonomy/cdm/model/common/TermBase.java
237 237
            Representation repr = getRepresentation(language);
238 238
            if (repr != null){
239 239
                repr.setLabel(label);
240
                this.titleCache = null;  //force titleCache refresh
241 240
            }else{
242 241
                repr = Representation.NewInstance(null, label, null, language);
243
            this.addRepresentation(repr);
242
                this.addRepresentation(repr);
243
            }
244
            this.titleCache = null; //force titleCache refresh
244 245
        }
245 246
    }
246
    }
247 247

  
248 248
    @Transient
249 249
    public String getDescription() {
cdmlib-model/src/main/java/eu/etaxonomy/cdm/strategy/parser/DeterminationModifierParser.java
1
/**
2
* Copyright (C) 2009 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.strategy.parser;
10

  
11
import org.apache.commons.lang.StringUtils;
12

  
13
import eu.etaxonomy.cdm.model.common.DefinedTerm;
14
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
15

  
16
/**
17
 * @author a.mueller
18
 * @date 28.10.2017
19
 */
20
public class DeterminationModifierParser {
21

  
22

  
23
//	public static final String qualifierPattern = "(?i)(aff|cf|vel. aff)\\.?";
24

  
25

  
26

  
27
	public static DefinedTerm parseDeterminationQualifier(String qualifier) throws UnknownCdmTypeException {
28

  
29
		if (StringUtils.isBlank(qualifier)){
30
			return null;
31
		}
32
		DefinedTerm term;
33
		if (qualifier.matches("aff(.?|inis)")){
34
			term = DefinedTerm.DETERMINATION_MODIFIER_AFFINIS();
35
		}else if (qualifier.matches("(cf.?|confer)")){
36
		    term = DefinedTerm.DETERMINATION_MODIFIER_CONFER();
37
		}else{
38
			String message = "Determination qualifier not supported: " + qualifier;
39
			throw new UnknownCdmTypeException(message);
40
		}
41
		return term;
42
	}
43
}
cdmlib-model/src/main/java/eu/etaxonomy/cdm/strategy/parser/SpecimenTypeParser.java
64 64
			status = SpecimenTypeDesignationStatus.NEOTYPE();
65 65
		}else if (type.equalsIgnoreCase("isoneo")){
66 66
			status = SpecimenTypeDesignationStatus.ISONEOTYPE();
67
		}else if (type.equalsIgnoreCase("type")){
67
		}else if (type.equalsIgnoreCase("para")){
68
            status = SpecimenTypeDesignationStatus.PARATYPE();
69
		}else if (type.equalsIgnoreCase("isopara")){
70
            status = SpecimenTypeDesignationStatus.ISOPARATYPE();
71
        }else if (type.equalsIgnoreCase("type")){
68 72
			status = SpecimenTypeDesignationStatus.TYPE();
69 73
		}else{
70 74
			String message = "Type Status not supported: " + type;
cdmlib-model/src/main/java/eu/etaxonomy/cdm/strategy/parser/TimePeriodParser.java
197 197
    private static void parseDateWithMonthName(String dateString, TimePeriod result) {
198 198
        String[] dates = dateString.split("(\\.|\\s+)+");
199 199

  
200
        Partial partial = new Partial();
201 200

  
202 201

  
203 202
        if (dates.length > 3 || dates.length < 2){
......
212 211
                Integer year = Integer.valueOf(strYear.trim());
213 212
                Integer month = monthNrFormName(strMonth.trim());
214 213
                Integer day = strDay == null ? null : Integer.valueOf(strDay.trim());
215
                //TODO deduplicate code with other routines
216
                if (year < 1000 && year > 2100){
217
                    logger.warn("Not a valid year: " + year + ". Year must be between 1000 and 2100");
218
                }else if (year < 1700 && year > 2100){
219
                    logger.warn("Not a valid taxonomic year: " + year + ". Year must be between 1750 and 2100");
220
                    partial = partial.with(TimePeriod.YEAR_TYPE, year);
221
                }else{
222
                    partial = partial.with(TimePeriod.YEAR_TYPE, year);
223
                }
224
                if (month != null && month != 0){
225
                    partial = partial.with(TimePeriod.MONTH_TYPE, month);
226
                }
227
                if (day != null && day != 0){
228
                    partial = partial.with(TimePeriod.DAY_TYPE, day);
229
                }
214

  
215
                Partial partial = makePartialFromDateParts(year, month, day);
230 216

  
231 217
                result.setStart(partial);
232 218
            } catch (IllegalArgumentException e) {
......
235 221
        }
236 222
    }
237 223

  
224

  
225
    /**
226
     * @param year
227
     * @param month
228
     * @param day
229
     * @return
230
     */
231
    public static Partial makePartialFromDateParts(Integer year, Integer month, Integer day) {
232
        Partial partial = new Partial();
233
        //TODO deduplicate code with other routines
234
        if (year < 1000 && year > 2100){
235
            logger.warn("Not a valid year: " + year + ". Year must be between 1000 and 2100");
236
        }else if (year < 1700 && year > 2100){
237
            logger.warn("Not a valid taxonomic year: " + year + ". Year must be between 1750 and 2100");
238
            partial = partial.with(TimePeriod.YEAR_TYPE, year);
239
        }else{
240
            partial = partial.with(TimePeriod.YEAR_TYPE, year);
241
        }
242
        if (month != null && month != 0){
243
            partial = partial.with(TimePeriod.MONTH_TYPE, month);
244
        }
245
        if (day != null && day != 0){
246
            partial = partial.with(TimePeriod.DAY_TYPE, day);
247
        }
248
        return partial;
249
    }
250

  
238 251
	/**
239 252
     * @param valueOf
240 253
     * @return
......
333 346
	}
334 347

  
335 348

  
349

  
336 350
	protected static Partial parseSingleDate(String singleDateString) throws IllegalArgumentException{
337 351
		//FIXME until now only quick and dirty and incomplete
338 352
		Partial partial =  new Partial();
......
395 409
			throw new IllegalArgumentException(e);
396 410
		}
397 411
		return partial;
398

  
399 412
	}
400 413

  
414

  
415
    /**
416
     * @see #parseEnglishDate(String, String, boolean)
417
     * @param strFrom the string representing the first part of the period
418
     * @param strTo the string representing the second part of the period
419
     * @return the parsed period
420
     */
421
    public static TimePeriod parseEnglishDate(String strFrom, String strTo) {
422
        return parseEnglishDate(strFrom, strTo, false);
423
    }
424

  
425
    /**
426
     * Parses 1 or 2 dates of format yyyy-mm-dd, where y, m and d are numbers.
427
     *
428
     * @param timePeriod
429
     * @param strFrom the string representing the first part of the period
430
     * @param strTo the string representing the second part of the period
431
     * @param isAmerican
432
     * @return the parsed period
433
     */
434
    private static TimePeriod parseEnglishDate(String strFrom, String strTo, boolean isAmerican) {
435
        Partial dateFrom = parseSingleEnglishDate(strFrom, isAmerican);
436
        Partial dateTo = parseSingleEnglishDate(strTo, isAmerican);
437
        TimePeriod result = TimePeriod.NewInstance(dateFrom, dateTo);
438
        return result;
439
    }
440

  
441

  
442
    private static final String ENGLISH_FORMAT = "\\d{4}-\\d{1,2}-\\d{1,2}";
443
    /**
444
     * @param strFrom
445
     * @return
446
     */
447
    private static Partial parseSingleEnglishDate(String strDate, boolean isAmerican) {
448
        if (StringUtils.isEmpty(strDate)){
449
            return null;
450
        }
451
        strDate = strDate.replace("\\s", "");
452
        if (!strDate.matches(ENGLISH_FORMAT)){
453
            throw new NumberFormatException("The given date is not of expected format yyyy-mm-dd: " + strDate);
454
        }
455
        String[] splits = strDate.split("-");
456
        Integer year = Integer.valueOf(splits[0]);
457
        Integer month = Integer.valueOf(splits[isAmerican? 2 : 1]);
458
        Integer day = Integer.valueOf(splits[isAmerican? 1 : 2]);
459
        //switch month and day if obvious
460
        if (month > 12 && day < 12){
461
            Integer tmp = month;
462
            month = day;
463
            day = tmp;
464
        }
465

  
466
        Partial result = makePartialFromDateParts(year, month, day);
467

  
468
        return result;
469
    }
470

  
401 471
}
cdmlib-persistence/src/main/java/eu/etaxonomy/cdm/persistence/dao/hibernate/common/DeduplicationHelper.java
177 177
			if (! reallocationPossible){
178 178
				String msg = "Merge not possible for objects of type %s and type %s";
179 179
				msg = String.format(msg, cdmBase1.getClass().getSimpleName(), cdmBase2.getClass().getSimpleName());
180
				throw new IllegalArgumentException("Merge not possible for objects of type %s and type %s");
180
				throw new IllegalArgumentException(msg);
181 181
			}else{
182 182
				if (! checkInstancesMergeable(cdmBase1, cdmBase2, strategy)){
183 183
					throw new IllegalArgumentException("Object can not be merged into new object as it is referenced in a way that does not allow merging");
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/facade/DerivedUnitFacade.java
111 111
		return new DerivedUnitFacade(type, null, null);
112 112
	}
113 113

  
114
	   /**
115
     * Creates a derived unit facade for a new derived unit of type
116
     * {@link SpecimenOrObservationType#PreservedSpecimen}.
117
     *
118
     * @return the derived unit facade
119
     */
120
    public static DerivedUnitFacade NewPreservedSpecimenInstance() {
121
        return new DerivedUnitFacade(SpecimenOrObservationType.PreservedSpecimen, null, null);
122
    }
123

  
114 124
	/**
115 125
	 * Creates a derived unit facade for a new derived unit of type
116 126
	 * <code>type</code>.

Also available in: Unified diff