这篇文章主要介绍了hadoop中如何实现DBInputFormat，具有一定借鉴价值，感兴趣的朋友可以参考下，希望大家阅读完这篇文章之后大有收获，下面让小编带着大家一起了解一下。
成都创新互联成立与2013年，是专业互联网技术服务公司，拥有项目网站设计制作、成都网站制作网站策划，项目实施与项目整合能力。我们以让每一个梦想脱颖而出为使命，1280元琼山做网站,已为上家服务,为琼山各地企业和个人服务,联系电话:18980820575
代码未做测试，先做记录
package com.test;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* 要运行本示例
* 1、把MySQL的jdbc驱动放到taskTracker的lib目录下，重启集群
*
*/
public class WordCountDB extends Configured implements Tool {
private String OUT_PATH = "hdfs://grid131:9000/output";
public static class Map extends Mapper {
public void map(LongWritable key, MyUser value, Context context) throws IOException, InterruptedException {
context.write(key, new Text(value.toString()));
}
}
public int run(String[] args) throws Exception {
Configuration conf = this.getConf();
DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://grid131:3306/test", "root", "admin");
//输出路径如果存在，则删除
FileSystem fs = FileSystem.get(new URI(OUT_PATH), conf);
fs.delete(new Path(OUT_PATH), true);
Job job = new Job(conf, WordCountDB.class.getSimpleName());
job.setJarByClass(WordCountDB.class);
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//指定不需要reduce，直接把map输出写入到hdfs中
job.setNumReduceTasks(0);
job.setInputFormatClass(DBInputFormat.class);
//指定表、字段
//DBInputFormat.setInput(job, inputClass, tableName, conditions, orderBy, fieldNames)
DBInputFormat.setInput(job, MyUser.class, "myuser", null, null, "id", "name");
job.setMapperClass(Map.class);
//当reduce输出类型与map输出类型一致时，map的输出类型可以不设置
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.waitForCompletion(true);
return job.isSuccessful()?0:1;
}
public static void main(String[] args) throws Exception {
int exit = ToolRunner.run(new WordCount(), args);
System.exit(exit);
}
}
class MyUser implements Writable, DBWritable {
private Long id;
private String name;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(this.id);
Text.writeString(out, this.name);
}
@Override
public void readFields(DataInput in) throws IOException {
this.id = in.readLong();
this.name = Text.readString(in);
}
@Override
public void write(PreparedStatement statement) throws SQLException {
statement.setLong(1, this.id);
statement.setString(2, this.name);
}
@Override
public void readFields(ResultSet resultSet) throws SQLException {
this.id = resultSet.getLong(1);
this.name = resultSet.getString(2);
}
@Override
public String toString() {
return this.id + "\t" + this.name;
}
} 感谢你能够认真阅读完这篇文章，希望小编分享的“hadoop中如何实现DBInputFormat”这篇文章对大家有帮助，同时也希望大家多多支持创新互联，关注创新互联行业资讯频道，更多相关知识等着你来学习!
