-
Notifications
You must be signed in to change notification settings - Fork 32
feat: add decoding as a multi-threaded CLARA engine #1229
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
38 commits
Select commit
Hold shift + click to select a range
833379b
add convenience method
baltzell 8743263
remove unused stuff
baltzell 9a115d1
add decoding engine
baltzell e8b29c0
add it to the clara yaml
baltzell 88a939d
use a pool
baltzell 36873f9
hmm
baltzell 784bc9f
allow decoder instances to share ConstantsManagers
baltzell 45c8279
share ConstantsManagers
baltzell b749b8a
kludge test
baltzell 9a613b1
cleanup
baltzell ce2f9e2
try this
baltzell c730204
inherit ConstantsManagers
baltzell ee44156
cleanup
baltzell 55f9bef
only check tables if not shared
baltzell b8ecc16
higher ports on macos, cleanup process dpe process
baltzell efc6ed2
remove ineffective pid trap
baltzell 0887a6f
Revert "higher ports on macos, cleanup process dpe process"
baltzell 2326b2d
Revert "fix job name"
baltzell e701d5a
Revert "decouple ubuntu/macos builds to reduce wait"
baltzell 178263b
Revert "remove example engine"
baltzell 103aea0
restore reported data type
baltzell fda4e97
rename class
baltzell f6555f2
undo
baltzell ed99e08
fix rebase oops
baltzell 6a8707f
rename class
baltzell c6fe641
fix rebase oops
baltzell 80c9c74
stf
baltzell 8765256
just events, not per thread
baltzell d15670c
reduce constants sharing
baltzell 9f2e529
bugfix
baltzell 403eb56
share one RCDBManager
baltzell 55dc634
bugfix
baltzell 80fbfd2
restore
baltzell 9ae21f8
DecoderEngine: just pass along HIPO events
baltzell 99587f3
cleanup, avoid class variable
baltzell 238ff63
remove debugging leftover
baltzell 47bf714
revert to hard-coded byte order
baltzell 4c42f10
default to no sharing
baltzell File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
common-tools/clas-reco/src/main/java/org/jlab/clas/reco/DecoderEngine.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| package org.jlab.clas.reco; | ||
|
|
||
| import java.util.Set; | ||
| import java.util.HashSet; | ||
| import java.nio.ByteBuffer; | ||
| import java.nio.ByteOrder; | ||
| import java.util.concurrent.ArrayBlockingQueue; | ||
| import java.util.concurrent.BlockingQueue; | ||
| import org.jlab.clara.base.ClaraUtil; | ||
| import org.jlab.clara.engine.Engine; | ||
| import org.jlab.clara.engine.EngineData; | ||
| import org.jlab.clara.engine.EngineDataType; | ||
| import org.jlab.clara.engine.EngineStatus; | ||
| import org.jlab.detector.decode.CLASDecoder; | ||
| import org.jlab.io.evio.EvioDataEvent; | ||
| import org.jlab.io.hipo.HipoDataEvent; | ||
| import org.jlab.jnp.hipo4.data.SchemaFactory; | ||
| import org.json.JSONObject; | ||
|
|
||
| /** | ||
| * | ||
| * @author baltzell | ||
| */ | ||
| public class DecoderEngine implements Engine { | ||
|
|
||
| static final int POOL_SIZE = 64; | ||
| static final Set<EngineDataType> ED_TYPES = ClaraUtil.buildDataTypes( | ||
| Clas12Types.EVIO,Clas12Types.HIPO,EngineDataType.JSON,EngineDataType.STRING); | ||
|
|
||
| SchemaFactory schema; | ||
| BlockingQueue<CLASDecoder> pool; | ||
| int constantsShared = 64; | ||
|
|
||
| public DecoderEngine() { | ||
| schema = new SchemaFactory(); | ||
| schema.initFromDirectory(System.getenv("CLAS12DIR") + "/etc/bankdefs/hipo4"); | ||
| } | ||
|
|
||
| @Override | ||
| public Set<EngineDataType> getInputDataTypes() { return ED_TYPES; } | ||
| @Override | ||
| public Set<EngineDataType> getOutputDataTypes() { return ED_TYPES; } | ||
| @Override | ||
| public EngineData executeGroup(Set<EngineData> set) { return null; } | ||
| @Override | ||
| public Set<String> getStates() { return new HashSet<>(); } | ||
| @Override | ||
| public String getDescription() { return "decoder engine"; } | ||
| @Override | ||
| public String getVersion() { return "1.0"; } | ||
| @Override | ||
| public String getAuthor() { return "baltzell"; } | ||
| @Override | ||
| public void reset() {} | ||
| @Override | ||
| public void destroy() {} | ||
|
|
||
| @Override | ||
| public EngineData configure(EngineData ed) { | ||
| JSONObject json = new JSONObject(ed.getData()); | ||
| pool = new ArrayBlockingQueue<>(POOL_SIZE); | ||
| CLASDecoder d0 = null; | ||
| for (int i=0; i<POOL_SIZE; i++) { | ||
| CLASDecoder d; | ||
| if (i % constantsShared == 0) { | ||
| d0 = new CLASDecoder(); | ||
| if (json.has("variation")) d0.setVariation(json.getString("variation")); | ||
| if (json.has("timestamp")) d0.setVariation(json.getString("timestamp")); | ||
| d = d0; | ||
| } | ||
| else { | ||
| d = new CLASDecoder(d0); | ||
| } | ||
| pool.add(d); | ||
| } | ||
| return ed; | ||
| } | ||
|
|
||
| @Override | ||
| public EngineData execute(EngineData input) { | ||
|
|
||
| EngineData output = input; | ||
|
|
||
| // if it's EVIO, decode it, otherwise just pass it along | ||
| if (input.getMimeType().equals("binary/data-evio")) { | ||
| EvioDataEvent evio; | ||
| try { | ||
| ByteBuffer bb = (ByteBuffer) input.getData(); | ||
| //evio = new EvioDataEvent(bb.array(), bb.order()); | ||
| evio = new EvioDataEvent(bb.array(), ByteOrder.LITTLE_ENDIAN); | ||
| } catch (Exception e) { | ||
| String msg = String.format("Error reading input event%n%n%s", ClaraUtil.reportException(e)); | ||
| output.setStatus(EngineStatus.ERROR); | ||
| output.setDescription(msg); | ||
| return output; | ||
| } | ||
| HipoDataEvent hipo; | ||
| try { | ||
| CLASDecoder d = pool.take(); | ||
| hipo = new HipoDataEvent(d.getDecodedEvent(evio),schema); | ||
| pool.put(d); | ||
| output.setData("binary/data-hipo", hipo.getHipoEvent()); | ||
| } catch (Exception e) { | ||
| String msg = String.format("Error processing input event%n%n%s", ClaraUtil.reportException(e)); | ||
| output.setStatus(EngineStatus.ERROR); | ||
| output.setDescription(msg); | ||
| return output; | ||
| } | ||
| } | ||
|
|
||
| return output; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.