Project

General

Profile

« Previous | Next » 

Revision e03ae1b2

Added by Andreas Müller over 4 years ago

ref #8479 rename taxon node import class

View differences:

cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/fauEu2Cdm/FauEu2CdmImport.java
1
/**
2
* Copyright (C) 2019 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.io.pesi.fauEu2Cdm;
10

  
11
import org.apache.log4j.Logger;
12
import org.springframework.stereotype.Component;
13
import org.springframework.transaction.TransactionStatus;
14

  
15
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
16
import eu.etaxonomy.cdm.io.common.ITaxonNodeOutStreamPartitioner;
17
import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitioner;
18
import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitionerConcurrent;
19
import eu.etaxonomy.cdm.model.taxon.Classification;
20
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
21

  
22
/**
23
 * @author a.mueller
24
 * @since 17.08.2019
25
 */
26
@Component
27
public class FauEu2CdmImport
28
        extends FauEu2CdmImportBase {
29

  
30
    private static final long serialVersionUID = -2111102574346601573L;
31
    private static final Logger logger = Logger.getLogger(FauEu2CdmImport.class);
32

  
33

  
34
    @Override
35
    protected void doInvoke(FauEu2CdmImportState state) {
36
        IProgressMonitor monitor = state.getConfig().getProgressMonitor();
37

  
38
        FauEu2CdmImportConfigurator config = state.getConfig();
39
        setRootId(state, config); //necessary?
40

  
41
        ITaxonNodeOutStreamPartitioner partitioner = getPartitioner(state, monitor, config);
42
        monitor.subTask("Start partitioning");
43
        doData(state, partitioner);
44
    }
45

  
46
    private void doData(FauEu2CdmImportState state, ITaxonNodeOutStreamPartitioner partitioner){
47
        TaxonNode node = partitioner.next();
48
        int partitionSize = 100;
49
        int count = 0;
50
        TransactionStatus tx = startTransaction();
51
        while (node != null) {
52
            node = doSingleNode(state, node);
53
            count++;
54
            if (count>=partitionSize){
55
                clearCache();
56
                try {
57
                    commitTransaction(tx);
58
                } catch (Exception e) {
59
                    logger.warn("Exception during commit node " + node.treeIndex());
60
                    e.printStackTrace();
61
                }
62
                tx = startTransaction();
63
                count=0;
64
            }
65
            node = partitioner.next();
66
        }
67
        commitTransaction(tx);
68
        partitioner.close();
69
    }
70

  
71
    private TaxonNode doSingleNode(FauEu2CdmImportState state, TaxonNode node) {
72
        TaxonNode result = null;
73
        logger.info(node.treeIndex());
74
        try {
75
            result = detache(node);
76
        } catch (Exception e) {
77
            logger.warn("Exception during detache node " + node.treeIndex());
78
            e.printStackTrace();
79
        }
80
        try {
81
            if (result != null){
82
                getTaxonNodeService().saveOrUpdate(node);
83
                getCommonService().saveOrUpdate(toSave);
84
                toSave.clear();
85
            }
86
        } catch (Exception e) {
87
            logger.warn("Exception during save node " + node.treeIndex());
88
             e.printStackTrace();
89
        }
90

  
91
        return result;
92
    }
93

  
94
    private ITaxonNodeOutStreamPartitioner getPartitioner(FauEu2CdmImportState state, IProgressMonitor monitor,
95
            FauEu2CdmImportConfigurator config) {
96
        ITaxonNodeOutStreamPartitioner partitioner = config.getPartitioner();
97
        if (partitioner == null){
98
            if(!config.isConcurrent()){
99
                partitioner = TaxonNodeOutStreamPartitioner.NewInstance(sourceRepo(state), state,
100
                        state.getConfig().getTaxonNodeFilter(), 100,
101
                        monitor, 1, TaxonNodeOutStreamPartitioner.fullPropertyPaths);
102
                ((TaxonNodeOutStreamPartitioner)partitioner).setLastCommitManually(true);
103
            }else{
104
                partitioner = TaxonNodeOutStreamPartitionerConcurrent
105
                        .NewInstance(state.getConfig().getSource(), state.getConfig().getTaxonNodeFilter(),
106
                                1000, monitor, 1, TaxonNodeOutStreamPartitioner.fullPropertyPaths);
107
            }
108
        }
109
        return partitioner;
110
    }
111

  
112
    private void setRootId(FauEu2CdmImportState state, FauEu2CdmImportConfigurator config) {
113
        if (config.getTaxonNodeFilter().hasClassificationFilter()) {
114
            Classification classification = getClassificationService()
115
                    .load(config.getTaxonNodeFilter().getClassificationFilter().get(0).getUuid());
116
            state.setRootId(classification.getRootNode().getUuid());
117
        } else if (config.getTaxonNodeFilter().hasSubtreeFilter()) {
118
            state.setRootId(config.getTaxonNodeFilter().getSubtreeFilter().get(0).getUuid());
119
        }
120
    }
121

  
122
    @Override
123
    protected boolean doCheck(FauEu2CdmImportState state) {
124
        return false;
125
    }
126

  
127
    @Override
128
    protected boolean isIgnore(FauEu2CdmImportState state) {
129
        return false;
130
    }
131

  
132
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/fauEu2Cdm/FauEu2CdmImportConfigurator.java
49 49
    @Override
50 50
    protected void makeIoClassList() {
51 51
        ioClassList = new Class[]{
52
                FauEu2CdmImport.class ,
52
                FauEu2CdmTaxonNodeImport.class ,
53 53
        };
54 54
    }
55 55

  
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/fauEu2Cdm/FauEu2CdmTaxonNodeImport.java
1
/**
2
* Copyright (C) 2019 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.io.pesi.fauEu2Cdm;
10

  
11
import org.apache.log4j.Logger;
12
import org.springframework.stereotype.Component;
13
import org.springframework.transaction.TransactionStatus;
14

  
15
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
16
import eu.etaxonomy.cdm.io.common.ITaxonNodeOutStreamPartitioner;
17
import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitioner;
18
import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitionerConcurrent;
19
import eu.etaxonomy.cdm.model.taxon.Classification;
20
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
21

  
22
/**
23
 * @author a.mueller
24
 * @since 17.08.2019
25
 */
26
@Component
27
public class FauEu2CdmTaxonNodeImport
28
        extends FauEu2CdmImportBase {
29

  
30
    private static final long serialVersionUID = -2111102574346601573L;
31
    private static final Logger logger = Logger.getLogger(FauEu2CdmTaxonNodeImport.class);
32

  
33

  
34
    @Override
35
    protected void doInvoke(FauEu2CdmImportState state) {
36
        IProgressMonitor monitor = state.getConfig().getProgressMonitor();
37

  
38
        FauEu2CdmImportConfigurator config = state.getConfig();
39
        setRootId(state, config); //necessary?
40

  
41
        ITaxonNodeOutStreamPartitioner partitioner = getPartitioner(state, monitor, config);
42
        monitor.subTask("Start partitioning");
43
        doData(state, partitioner);
44
    }
45

  
46
    private void doData(FauEu2CdmImportState state, ITaxonNodeOutStreamPartitioner partitioner){
47
        TaxonNode node = partitioner.next();
48
        int partitionSize = 100;
49
        int count = 0;
50
        TransactionStatus tx = startTransaction();
51
        while (node != null) {
52
            node = doSingleNode(state, node);
53
            count++;
54
            if (count>=partitionSize){
55
                clearCache();
56
                try {
57
                    commitTransaction(tx);
58
                } catch (Exception e) {
59
                    logger.warn("Exception during commit node " + node.treeIndex());
60
                    e.printStackTrace();
61
                }
62
                tx = startTransaction();
63
                count=0;
64
            }
65
            node = partitioner.next();
66
        }
67
        commitTransaction(tx);
68
        partitioner.close();
69
    }
70

  
71
    private TaxonNode doSingleNode(FauEu2CdmImportState state, TaxonNode node) {
72
        TaxonNode result = null;
73
        logger.info(node.treeIndex());
74
        try {
75
            result = detache(node);
76
        } catch (Exception e) {
77
            logger.warn("Exception during detache node " + node.treeIndex());
78
            e.printStackTrace();
79
        }
80
        try {
81
            if (result != null){
82
                getTaxonNodeService().saveOrUpdate(node);
83
                getCommonService().saveOrUpdate(toSave);
84
                toSave.clear();
85
            }
86
        } catch (Exception e) {
87
            logger.warn("Exception during save node " + node.treeIndex());
88
             e.printStackTrace();
89
        }
90

  
91
        return result;
92
    }
93

  
94
    private ITaxonNodeOutStreamPartitioner getPartitioner(FauEu2CdmImportState state, IProgressMonitor monitor,
95
            FauEu2CdmImportConfigurator config) {
96
        ITaxonNodeOutStreamPartitioner partitioner = config.getPartitioner();
97
        if (partitioner == null){
98
            if(!config.isConcurrent()){
99
                partitioner = TaxonNodeOutStreamPartitioner.NewInstance(sourceRepo(state), state,
100
                        state.getConfig().getTaxonNodeFilter(), 100,
101
                        monitor, 1, TaxonNodeOutStreamPartitioner.fullPropertyPaths);
102
                ((TaxonNodeOutStreamPartitioner)partitioner).setLastCommitManually(true);
103
            }else{
104
                partitioner = TaxonNodeOutStreamPartitionerConcurrent
105
                        .NewInstance(state.getConfig().getSource(), state.getConfig().getTaxonNodeFilter(),
106
                                1000, monitor, 1, TaxonNodeOutStreamPartitioner.fullPropertyPaths);
107
            }
108
        }
109
        return partitioner;
110
    }
111

  
112
    private void setRootId(FauEu2CdmImportState state, FauEu2CdmImportConfigurator config) {
113
        if (config.getTaxonNodeFilter().hasClassificationFilter()) {
114
            Classification classification = getClassificationService()
115
                    .load(config.getTaxonNodeFilter().getClassificationFilter().get(0).getUuid());
116
            state.setRootId(classification.getRootNode().getUuid());
117
        } else if (config.getTaxonNodeFilter().hasSubtreeFilter()) {
118
            state.setRootId(config.getTaxonNodeFilter().getSubtreeFilter().get(0).getUuid());
119
        }
120
    }
121

  
122
    @Override
123
    protected boolean doCheck(FauEu2CdmImportState state) {
124
        return false;
125
    }
126

  
127
    @Override
128
    protected boolean isIgnore(FauEu2CdmImportState state) {
129
        return false;
130
    }
131

  
132
}

Also available in: Unified diff