how to create yarn application #cmdevio2017

38

Upload: johnsmith

Post on 24-Jan-2018

472 views

Category:

Engineering


3 download

TRANSCRIPT

Page 1: How to create Yarn Application #cmdevio2017
Page 2: How to create Yarn Application #cmdevio2017
Page 3: How to create Yarn Application #cmdevio2017
Page 4: How to create Yarn Application #cmdevio2017
Page 5: How to create Yarn Application #cmdevio2017
Page 6: How to create Yarn Application #cmdevio2017
Page 7: How to create Yarn Application #cmdevio2017
Page 8: How to create Yarn Application #cmdevio2017
Page 9: How to create Yarn Application #cmdevio2017
Page 10: How to create Yarn Application #cmdevio2017
Page 11: How to create Yarn Application #cmdevio2017
Page 12: How to create Yarn Application #cmdevio2017
Page 13: How to create Yarn Application #cmdevio2017
Page 14: How to create Yarn Application #cmdevio2017
Page 15: How to create Yarn Application #cmdevio2017
Page 16: How to create Yarn Application #cmdevio2017

YarnClient client = YarnClient.createYarnClient();Configuration conf = new Configuration();client.init(conf);client.start();

YarnClientApplication app = client.createApplication();ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();ApplicationId appId = appContext.getApplicationId();

Page 17: How to create Yarn Application #cmdevio2017

// appContext.setApplicationName(appName);

// AM memory, vcore Resource resource = Records.newRecord(Resource.class);resource.setMemory(amMemory);resource.setVirtualCores(amVcore);appContext.setResource(resource);

Page 18: How to create Yarn Application #cmdevio2017

Map<String, LocalResource> localResources = new HashMap<>();ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class);

FileSystem fs = FileSystem.get(conf);

// HDFS Jarfs.copyFromLocalFile(new Path(src), new Path(distPath));

// JarFileStatus status = fs.getFileStatus(distPath);LocalResource localResource = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dist.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, status.getLen(), status.getModificationTime());localResources.put(distJar, localResource);container.setLocalResources(localResources);

Page 19: How to create Yarn Application #cmdevio2017

Map<String, String> env = new HashMap<>();LocalResource appJar = localResources.get(distJar);Path jarPath= new Path(distPath);

env.put(“jar_path”, distPath);env.put(“jar_length”, Long.toString(status.getLen()));env.put(“jar_timestamp”, Long.toString(status.getModificationTime()));

StringBuilder classPath = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");

// ※ Hadoop classpath

env.put(“classpath”, classPath.toString());container.setEnviroment(env);

Page 20: How to create Yarn Application #cmdevio2017

StringBuilder sb = new StringBuilder();sb.appned(Environment.JAVA_HOME.$$() + “/bin/java ”) .append("-Xmx" + amMemory + "m ") .append("jp.classmethod.yarn_app.MyApplicationMaster ") .append("…") // .append("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout ") .append("2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout ");List<String> cmd = new ArrayList<>();cmd.append(sb.toString());container.setCommands(cmd);appContext.setAMContainerSpec(container);

Page 21: How to create Yarn Application #cmdevio2017

client.submitApplication();

Page 22: How to create Yarn Application #cmdevio2017

while(true) { ApplicationReport report = client.getApplicationReport(appId); YarnApplicationState appState = report.getYarnApplicationState(); FinalApplicationStatus finStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCESS == finStatus) { // } else { // } Thread.sleep(1000);}

Page 23: How to create Yarn Application #cmdevio2017
Page 24: How to create Yarn Application #cmdevio2017
Page 25: How to create Yarn Application #cmdevio2017
Page 26: How to create Yarn Application #cmdevio2017

Configuration conf = new Configuration();

// (ex. attempId )ContainerId containerId = ConverterUtils.toContainerId( envs.get(ApplicationConstants.Environment.CONTAINER_ID.name()));ApplicationAttemptId attemptId = containerId.getApplicationAttemptId();

Page 27: How to create Yarn Application #cmdevio2017

AMRMClient<ContainerRequest> client = AMRMClient.createAMRMClient();client.init(conf);client.start();

// ApplicationMasterclient.registerApplicationMaster("", 0, "");

// for (i = 0; i < totalContainerSize; i++) { ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority); client.addContainerRequest(containerReq);}

Page 28: How to create Yarn Application #cmdevio2017

NMClient nmClient = NMClient.createNMClient();nmClient.init(conf);nmClient.start();

// //int allocatedContainers = 0;int complitedConttainers = 0;

Page 29: How to create Yarn Application #cmdevio2017

while(completedContainers < totalContainerSize) { AllocateResponse response = amRMClient.allocate(completedContainers / totalContainerSize);

for (Container container : response.getAllocatedContainers()) { allocatedContainers++; ContainerLaunchContext appContainer = … // ※ container nmClient.startContainer(container, appContainer);

}

completedContainers += response.getCompletedContainersStatuses().size();

Thread.sleep(100);}

Page 30: How to create Yarn Application #cmdevio2017

ContainerLaunchContext appContainer = Records.newRecord(ContainerLaunchContext.class);

// jarappContainer.setLocalRecources( Collections.singletonMap("appJar", appJar);appContainer.setEnviroment(env);

// Container javaappContainer.setCommands("$JAVA_HOME/bin/java" + "…");

Page 31: How to create Yarn Application #cmdevio2017

// client.unregisterApplicationMaster( FinalApplicationStatus.SUCCEEDED, "", "");

Page 32: How to create Yarn Application #cmdevio2017
Page 33: How to create Yarn Application #cmdevio2017
Page 34: How to create Yarn Application #cmdevio2017
Page 35: How to create Yarn Application #cmdevio2017
Page 37: How to create Yarn Application #cmdevio2017
Page 38: How to create Yarn Application #cmdevio2017