功能：
定期去掃描元數(shù)據(jù)表CoordinatorJobBean，將滿(mǎn)足條件的Job實(shí)體化；
// default is 300sec (5min)
int schedulingInterval = Services.get().getConf().getInt(CONF_SCHEDULING_INTERVAL, lookupInterval);
Runnable lookupTriggerJobsRunnable = new CoordMaterializeTriggerRunnable(materializationWindow, lookupInterval);
services.get(SchedulerService.class).schedule(lookupTriggerJobsRunnable, 10, schedulingInterval,SchedulerService.Unit.SEC);
CoordMaterializeTriggerRunnable:
oozie里面的觸發(fā)機(jī)制：
1、啟動(dòng)一個(gè)定時(shí)執(zhí)行器，每隔 schedulingInterval 時(shí)間運(yùn)行一次 CoordMaterializeTriggerRunnable；
2、查詢(xún)CoordinatorJobBean根據(jù)時(shí)間和條數(shù)限制獲取本次需要實(shí)例化的CoordinatorJobBean；
技巧： 在CoordinatorJobBean中有個(gè)字段 nextMaterializedTimestamp 表明，在這個(gè)時(shí)間點(diǎn)之前的實(shí)例已經(jīng)存在，為了盡可能的讓任務(wù)的實(shí)例盡可能在觸發(fā)之前
實(shí)例就產(chǎn)生，而不是事后產(chǎn)生，在查詢(xún)CoordinatorJobBean使用的時(shí)候不是使用當(dāng)前時(shí)間，而是使用未來(lái)的一個(gè)時(shí)間： new Date().getTime() + lookupInterval * 1000
3、異步去實(shí)例化 CoordinatorJobBean 下的 action；
4、在實(shí)例化action的時(shí)候，生成的是窗口期內(nèi)的一批實(shí)例，在類(lèi) CoordMaterializeTransitionXCommand中實(shí)現(xiàn)；
5、loadState()：加載 CoordinatorJobBean 信息，計(jì)算這次要實(shí)例化的時(shí)間窗口： startMatdTime 和 endMatdTime；
6、materialize()：根據(jù)不同的計(jì)算方式(cron、自定義)來(lái)在這個(gè)時(shí)間窗口內(nèi)循環(huán)的產(chǎn)生action的實(shí)例；
7、更新 CoordinatorJobBean 狀態(tài)，記錄 endMatdTime、lastActionNumber（時(shí)間累加）、狀態(tài)置為running狀態(tài)、如果jobEndTime< endMatdTime
說(shuō)明這個(gè) CoordinatorJobBean 實(shí)例化工作已經(jīng)結(jié)束，標(biāo)記 job.setDoneMaterialization();
8、performWrites():更新數(shù)據(jù)庫(kù)
9、notifyParent():通知上層結(jié)構(gòu) bundle;
/** * This runnable class will run in every "interval" to queue CoordMaterializeTransitionXCommand.
*/
static class CoordMaterializeTriggerRunnable implements Runnable {
private int materializationWindow;
private int lookupInterval;
private long delay = 0;
private List<XCallable<Void>> callables;
private List<XCallable<Void>> delayedCallables;
private XLog LOG = XLog.getLog(getClass());
public CoordMaterializeTriggerRunnable(int materializationWindow, int lookupInterval) {
this.materializationWindow = materializationWindow;
this.lookupInterval = lookupInterval;
}
@Override
public void run() {
LockToken lock = null;
// first check if there is some other running instance from the same service;
try {
lock = Services.get().get(MemoryLocksService.class)
.getWriteLock(CoordMaterializeTriggerService.class.getName(), lockTimeout);
if (lock != null) {
runCoordJobMatLookup();
if (null != callables) {
boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
if (ret == false) {
XLog.getLog(getClass()).warn(
"Unable to queue the callables commands for CoordMaterializeTriggerRunnable. "
+ "Most possibly command queue is full. Queue size is :"
+ Services.get().get(CallableQueueService.class).queueSize());
}
callables = null;
}
if (null != delayedCallables) {
boolean ret = Services.get().get(CallableQueueService.class)
.queueSerial(delayedCallables, this.delay);
if (ret == false) {
XLog.getLog(getClass()).warn(
"Unable to queue the delayedCallables commands for CoordMaterializeTriggerRunnable. "
+ "Most possibly Callable queue is full. Queue size is :"
+ Services.get().get(CallableQueueService.class).queueSize());
}
delayedCallables = null;
this.delay = 0;
}
}
else {
LOG.debug("Can't obtain lock, skipping");
}
}
catch (Exception e) {
LOG.error("Exception", e);
}
finally {
if (lock != null) {
lock.release();
LOG.info("Released lock for [{0}]", CoordMaterializeTriggerService.class.getName());
}
}
}
/**
* Recover coordinator jobs that should be materialized
* @throws JPAExecutorException
*/
private void runCoordJobMatLookup() throws JPAExecutorException {
List<UpdateEntry> updateList = new ArrayList<UpdateEntry>(); XLog.Info.get().clear();
XLog LOG = XLog.getLog(getClass());
try {
// get current date
Date currDate = new Date(new Date().getTime() + lookupInterval * 1000);
// get list of all jobs that have actions that should be materialized.
int materializationLimit = ConfigurationService.getInt(CONF_MATERIALIZATION_SYSTEM_LIMIT);
materializeCoordJobs(currDate, materializationLimit, LOG, updateList);
}
catch (Exception ex) {
LOG.error("Exception while attempting to materialize coordinator jobs, {0}", ex.getMessage(), ex);
}
finally {
BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, null);
}
}
private void materializeCoordJobs(Date currDate, int limit, XLog LOG, List<UpdateEntry> updateList)
throws JPAExecutorException {
try {
List<CoordinatorJobBean> materializeJobs = CoordJobQueryExecutor.getInstance().getList(
CoordJobQuery.GET_COORD_JOBS_OLDER_FOR_MATERIALIZATION, currDate, limit);
LOG.info("CoordMaterializeTriggerService - Curr Date= " + DateUtils.formatDateOozieTZ(currDate)
+ ", Num jobs to materialize = " + materializeJobs.size());
for (CoordinatorJobBean coordJob : materializeJobs) {
Services.get().get(InstrumentationService.class).get()
.incr(INSTRUMENTATION_GROUP, INSTR_MAT_JOBS_COUNTER, 1);
queueCallable(new CoordMaterializeTransitionXCommand(coordJob.getId(), materializationWindow));
coordJob.setLastModifiedTime(new Date());
updateList.add(new UpdateEntry<CoordJobQuery>(CoordJobQuery.UPDATE_COORD_JOB_LAST_MODIFIED_TIME,
coordJob));
}
}
catch (JPAExecutorException jex) {
LOG.warn("JPAExecutorException while attempting to materialize coordinator jobs", jex);
}
}
/**
* Adds callables to a list. If the number of callables in the list reaches {@link
* CoordMaterializeTriggerService#CONF_CALLABLE_BATCH_SIZE}, the entire batch is queued and the callables list
* is reset.
*
* @param callable the callable to queue.
*/
private void queueCallable(XCallable<Void> callable) {
if (callables == null) {
callables = new ArrayList<XCallable<Void>>();
}
callables.add(callable);
if (callables.size() == ConfigurationService.getInt(CONF_CALLABLE_BATCH_SIZE)) { boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
if (ret == false) {
XLog.getLog(getClass()).warn(
"Unable to queue the callables commands for CoordMaterializeTriggerRunnable. "
+ "Most possibly command queue is full. Queue size is :"
+ Services.get().get(CallableQueueService.class).queueSize());
}
callables = new ArrayList<XCallable<Void>>();
}
}
}
