为什么运行mapreduce时一直空指针异常。
“C:\Program Files\Java\jdk1.8.0_231\bin\java.exe” “-javaagent:E:\IntelliJ IDEA 2019.3.1\lib\idea_rt.jar=57147:E:\IntelliJ IDEA 2019.3.1\bin” -Dfile.encoding=UTF-8 -classpath “C:\Program Files\Java\jdk1.8.0_231\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\cldrdata.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\jfxrt.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\nashorn.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\sunpkcs11.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\jce.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\jfxswt.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\resources.jar;C:\Program Files\Java\jdk1.8.0_231\jre\lib\rt.jar;E:\WordCount\target\classes;D:\repository\org\apache\hadoop\hadoop-client\2.7.1\hadoop-client-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-common\2.7.1\hadoop-common-2.7.1.jar;D:\repository\com\google\guava\guava\11.0.2\guava-11.0.2.jar;D:\repository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;D:\repository\org\apache\commons\commons-math3\3.1.1\commons-math3-3.1.1.jar;D:\repository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;D:\repository\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;D:\repository\commons-codec\commons-codec\1.4\commons-codec-1.4.jar;D:\repository\commons-io\commons-io\2.4\commons-io-2.4.jar;D:\repository\commons-net\commons-net\3.1\commons-net-3.1.jar;D:\repository\commons-collections\commons-collections\3.2.1\commons-collections-3.2.1.jar;D:\repository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;D:\repository\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;D:\repository\log4j\log4j\1.2.17\log4j-1.2.17.jar;D:\repository\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;D:\repository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;D:\repository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;D:\repository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;D:\repository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;D:\repository\org\slf4j\slf4j-api\1.7.10\slf4j-api-1.7.10.jar;D:\repository\org\slf4j\slf4j-log4j12\1.7.10\slf4j-log4j12-1.7.10.jar;D:\repository\org\codehaus\jackson\jackson-core-asl\1.9.13\jackson-core-asl-1.9.13.jar;D:\repository\org\codehaus\jackson\jackson-mapper-asl\1.9.13\jackson-mapper-asl-1.9.13.jar;D:\repository\org\apache\avro\avro\1.7.4\avro-1.7.4.jar;D:\repository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;D:\repository\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;D:\repository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;D:\repository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;D:\repository\org\apache\hadoop\hadoop-auth\2.7.1\hadoop-auth-2.7.1.jar;D:\repository\org\apache\httpcomponents\httpclient\4.2.5\httpclient-4.2.5.jar;D:\repository\org\apache\httpcomponents\httpcore\4.2.4\httpcore-4.2.4.jar;D:\repository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;D:\repository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;D:\repository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;D:\repository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;D:\repository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;D:\repository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;D:\repository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;D:\repository\com\google\code\findbugs\jsr305\3.0.0\jsr305-3.0.0.jar;D:\repository\org\apache\htrace\htrace-core\3.1.0-incubating\htrace-core-3.1.0-incubating.jar;D:\repository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;D:\repository\io\netty\netty\3.7.0.Final\netty-3.7.0.Final.jar;D:\repository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;D:\repository\org\tukaani\xz\1.0\xz-1.0.jar;D:\repository\org\apache\hadoop\hadoop-hdfs\2.7.1\hadoop-hdfs-2.7.1.jar;D:\repository\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;D:\repository\io\netty\netty-all\4.0.23.Final\netty-all-4.0.23.Final.jar;D:\repository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;D:\repository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;D:\repository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-app\2.7.1\hadoop-mapreduce-client-app-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-common\2.7.1\hadoop-mapreduce-client-common-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-yarn-client\2.7.1\hadoop-yarn-client-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-yarn-server-common\2.7.1\hadoop-yarn-server-common-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.7.1\hadoop-mapreduce-client-shuffle-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-yarn-api\2.7.1\hadoop-yarn-api-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-core\2.7.1\hadoop-mapreduce-client-core-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-yarn-common\2.7.1\hadoop-yarn-common-2.7.1.jar;D:\repository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;D:\repository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;D:\repository\javax\activation\activation\1.1\activation-1.1.jar;D:\repository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;D:\repository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;D:\repository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;D:\repository\org\codehaus\jackson\jackson-jaxrs\1.9.13\jackson-jaxrs-1.9.13.jar;D:\repository\org\codehaus\jackson\jackson-xc\1.9.13\jackson-xc-1.9.13.jar;D:\repository\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.7.1\hadoop-mapreduce-client-jobclient-2.7.1.jar;D:\repository\org\apache\hadoop\hadoop-annotations\2.7.1\hadoop-annotations-2.7.1.jar” WordCountApp
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread “main” java.lang.NullPointerException
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:483)
at org.apache.hadoop.util.Shell.run(Shell.java:456)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:722)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:815)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:798)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:728)
at org.apache.hadoop.fs.RawLocalFileSystem.mkOneDirWithMode(RawLocalFileSystem.java:486)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:527)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:504)
at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:305)
at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:133)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:144)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at WordCountApp.main(WordCountApp.java:35)
代码：
public class WordCountApp {
public static void main(String[] args) throws Exception {
//System.setProperty("HADOOP_USER_NAME","hadoop");
Configuration conf=new Configuration();
//conf.set("fs.defaultFS","hdfs://192.168.126.130:8020");
Job job=Job.getInstance(conf);
//job的主类
job.setJarByClass(WordCountApp.class);
//job的处理类
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReduce.class);
//map输出的key value的类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//reduce输出的key value 的类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.setInputPaths(job,new Path("input"));
FileOutputFormat.setOutputPath(job,new Path("output"));
boolean result=job.waitForCompletion(true);
System.out.println(result);
}
public class WordCountReduce extends Reducer<Text, IntWritable,Text,IntWritable> {
/**
* map的输出到reduce的输入，相同的key分发到一个reduce上去
* reduce1 （hello 1） （hello 1）==> (hello <1,1>)
* @param
* @param values
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int count =0;
Iterator<IntWritable> iterator=values.iterator();
while(iterator.hasNext()){
IntWritable value=iterator.next();
count+=value.get();
}
context.write(key,new IntWritable(count));
}
}
public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] words=value.toString().split(" ");
for(String word:words){
context.write(new Text(word),new IntWritable(1));
}
}
}
正在回答
相似问题
登录后可查看更多问答，登录/注册
