hadoop修改配置后如何动态生效

转载

小题大作 2024-10-30 16:45:08

文章标签 hadoop修改配置后如何动态生效大数据 Text ci 脏数据 文章分类 Hadoop 大数据

最近一直在做地域统计的功能，用户下载app的日志中记录了IP，老大要根据这个IP做个地域统计，看看哪些地方的用户喜欢下载哪些应用。最初是用Java跑纯真IP地址数据库，然后对日志进行统计，不过效率不高，20+M的数据要跑几个小时，后来就把任务给我了，让我用Hadoop跑跑试试。

1. ip的解析，把ip地址解析成实际地址：使用了纯真ip数据库QQwry.dat

参考代码：

2.新建一个hadoop项目，将ip解析的3个类放到一个包下面

3.QQwry的测试使用：项目中需要拿到ip的国家和省，然后追加到原始日志作为2个新字段，但是测试发现返回的数据不能直接拿来处理，所有就需要进一步的格式化处理。

@1.对于：内蒙，广西，新疆，宁夏，西藏这5个自治区，直接返回中国+自治区名

@2，包含省字的，直接返回中国+省份，

@3，包含市的，主要是上海，北京，重庆，天津4个直辖市，返回中国+城市

@5，包含中国的，直接返回中国，省份字段留空；

@6.其他就直接返回解析的数据，省份留空 (基本就是国外的)

@4，测试中发现这样处理不干净，还有很多脏数据，例如包含大学和学院的，还有欧洲中部，XX网吧等。所有需要对这些脏数据进行处理。采用的方法是将脏数据按一定格式保存到dirtydata.txt中，然后初始化到一个map中，利用map进行格式化。

4.根据3中的格式化需求，编写格式化函数

private String formatCity(String country) {
	// 特殊地区处理，
	for (String spe : spelist) {
		if (country.indexOf(spe) != -1)
			return "中国," + spe;
	}
	if (country.indexOf("省") != -1) {
		String contrysplit[] = country.split("省");
		return "中国," + contrysplit[0] + "省";
	else if (country.indexOf("市") != -1) {
		String citysplist[] = country.split("市");
		return "中国," + citysplist[0] + "市";
	} else if (umap.containsKey(country)) {
		eturn "中国," + umap.get(country);
	} else if (country.indexOf("中国") != -1) {
		return "中国," + "";
	} else {
		return country + "," + "";
	}
}
private String formatCity(String country) {
	// 特殊地区处理，
	for (String spe : spelist) {
		if (country.indexOf(spe) != -1)
			return "中国," + spe;
	}
	if (country.indexOf("省") != -1) {
		String contrysplit[] = country.split("省");
		return "中国," + contrysplit[0] + "省";
	else if (country.indexOf("市") != -1) {
		String citysplist[] = country.split("市");
		return "中国," + citysplist[0] + "市";
	} else if (umap.containsKey(country)) {
		eturn "中国," + umap.get(country);
	} else if (country.indexOf("中国") != -1) {
		return "中国," + "";
	} else {
		return country + "," + "";
	}
}

5.对于脏数据的解析，读取txt文件，然后编写解析函数
public Map<String, String> getUniversMap(String filepath)
				throws FileNotFoundException {
	Map<String, String> universMap = new HashMap<String, String>();
	FileReader fr = new FileReader(filepath);
	BufferedReader br = new BufferedReader(fr);
	String readoneline;
	String tmp[];
	try {
		while ((readoneline = br.readLine()) != null) {
		    tmp = readoneline.split(",");
		    if(tmp.length == 3){
			universMap.put(tmp[0], tmp[2]);
		    }
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
	return universMap;
}
5.对于脏数据的解析，读取txt文件，然后编写解析函数
public Map<String, String> getUniversMap(String filepath)
				throws FileNotFoundException {
	Map<String, String> universMap = new HashMap<String, String>();
	FileReader fr = new FileReader(filepath);
	BufferedReader br = new BufferedReader(fr);
	String readoneline;
	String tmp[];
	try {
		while ((readoneline = br.readLine()) != null) {
		    tmp = readoneline.split(",");
		    if(tmp.length == 3){
			universMap.put(tmp[0], tmp[2]);
		    }
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
	return universMap;
}

6.编写map/reduce程序，调用1中的函数，然后对结果进行格式化

public class ConvertIp {

	public static class ItemMapper extends
			Mapper<Object, Text, Text, NullWritable> {
		private Text outkey = new Text("");
		private IPSeeker ipSeeker;
		private String filepath;
		Map<String, String> umap;
		final String spelist[] = { "内蒙古", "广西", "新疆", "宁夏", "西藏" };
		protected void map(Object key, Text value, Context context)
				throws IOException, InterruptedException {

			String line = value.toString();
			String details[] = line.split(",");
			if(details.length != 15){
				return;
			}
			String ip = details[3];
			String reg = "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})";
			// data clien 1.length=15 2.the university
			if (ip.matches(reg)) {
				outkey.set(new StringBuffer().append(line).append(
						",").append(formatCity(ipSeeker.getCountry(ip)))
						.toString());

				context.write(outkey, NullWritable.get());
			}
		}

		@Override
		protected void setup(Context context) throws IOException,
				InterruptedException {
			ipSeeker = new IPSeeker("qqwry.dat");  //初始化，这种写法需要在执行hadoop命令时
使用-files +qqwry.dat的路径,+dirtydata.txt的路径（上传到job的临时目录下）
                        filepath = "dirtydata.txt";         //初始化
			try {
				umap = getUniversMap(filepath);
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			}
			super.setup(context);
		}

		private String formatCity(String country) {
			for (String spe : spelist) {
				if (country.indexOf(spe) != -1)
					return "中国," + spe;
			}
			if (country.indexOf("省") != -1) {
				String contrysplit[] = country.split("省");
				return "中国," + contrysplit[0] + "省";
			} else if (country.indexOf("市") != -1) {
				String citysplist[] = country.split("市");
				return "中国," + citysplist[0] + "市";
			} else if (umap.containsKey(country)) {
				return "中国," + umap.get(country);
			} else if (country.indexOf("中国") != -1) {
				return "中国," + "";
			} else {
				return country + "," + "";
			}
		}

		public Map<String, String> getUniversMap(String filepath)
				throws FileNotFoundException {
			Map<String, String> universMap = new HashMap<String, String>();
			FileReader fr = new FileReader(filepath);
			BufferedReader br = new BufferedReader(fr);
			String readoneline;
			String tmp[];
			try {
				while ((readoneline = br.readLine()) != null) {
					tmp = readoneline.split(",");
					if(tmp.length == 3){
						universMap.put(tmp[0], tmp[2]);
					}
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			return universMap;
		}
	}

	public static class ItemReducer extends
			Reducer<Text, NullWritable, Text, NullWritable> {

		protected void reduce(Text key, Iterable<NullWritable> values,
				Context context) throws IOException, InterruptedException {
			context.write(key, NullWritable.get());
		}

	}

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf, args)
				.getRemainingArgs();
		if (otherArgs.length != 2) {
			System.err.println("Usage: -file <file> <in> <out>");
			System.exit(2);
		}

		Job job = new Job(conf, "ipcount");
		job.setJarByClass(ConvertIp.class);
		job.setMapperClass(ItemMapper.class);
		job.setReducerClass(ItemReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}
public class ConvertIp {

	public static class ItemMapper extends
			Mapper<Object, Text, Text, NullWritable> {
		private Text outkey = new Text("");
		private IPSeeker ipSeeker;
		private String filepath;
		Map<String, String> umap;
		final String spelist[] = { "内蒙古", "广西", "新疆", "宁夏", "西藏" };
		protected void map(Object key, Text value, Context context)
				throws IOException, InterruptedException {

			String line = value.toString();
			String details[] = line.split(",");
			if(details.length != 15){
				return;
			}
			String ip = details[3];
			String reg = "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})";
			// data clien 1.length=15 2.the university
			if (ip.matches(reg)) {
				outkey.set(new StringBuffer().append(line).append(
						",").append(formatCity(ipSeeker.getCountry(ip)))
						.toString());

				context.write(outkey, NullWritable.get());
			}
		}

		@Override
		protected void setup(Context context) throws IOException,
				InterruptedException {
			ipSeeker = new IPSeeker("qqwry.dat");  //初始化，这种写法需要在执行hadoop命令时
使用-files +qqwry.dat的路径,+dirtydata.txt的路径（上传到job的临时目录下）
                        filepath = "dirtydata.txt";         //初始化
			try {
				umap = getUniversMap(filepath);
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			}
			super.setup(context);
		}

		private String formatCity(String country) {
			for (String spe : spelist) {
				if (country.indexOf(spe) != -1)
					return "中国," + spe;
			}
			if (country.indexOf("省") != -1) {
				String contrysplit[] = country.split("省");
				return "中国," + contrysplit[0] + "省";
			} else if (country.indexOf("市") != -1) {
				String citysplist[] = country.split("市");
				return "中国," + citysplist[0] + "市";
			} else if (umap.containsKey(country)) {
				return "中国," + umap.get(country);
			} else if (country.indexOf("中国") != -1) {
				return "中国," + "";
			} else {
				return country + "," + "";
			}
		}

		public Map<String, String> getUniversMap(String filepath)
				throws FileNotFoundException {
			Map<String, String> universMap = new HashMap<String, String>();
			FileReader fr = new FileReader(filepath);
			BufferedReader br = new BufferedReader(fr);
			String readoneline;
			String tmp[];
			try {
				while ((readoneline = br.readLine()) != null) {
					tmp = readoneline.split(",");
					if(tmp.length == 3){
						universMap.put(tmp[0], tmp[2]);
					}
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			return universMap;
		}
	}

	public static class ItemReducer extends
			Reducer<Text, NullWritable, Text, NullWritable> {

		protected void reduce(Text key, Iterable<NullWritable> values,
				Context context) throws IOException, InterruptedException {
			context.write(key, NullWritable.get());
		}

	}

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf, args)
				.getRemainingArgs();
		if (otherArgs.length != 2) {
			System.err.println("Usage: -file <file> <in> <out>");
			System.exit(2);
		}

		Job job = new Job(conf, "ipcount");
		job.setJarByClass(ConvertIp.class);
		job.setMapperClass(ItemMapper.class);
		job.setReducerClass(ItemReducer.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}

7.打包成jar，上传到服务器，然后进行测试

hadoop jar /home/wjk/ipcount.jar com.wjk.datastat.ip.ConvertIp -files /home/wjk/qqwry.dat,/home/wjk/dirtydata.txt  /user/hive/warehouse/active_log/dt=20120301  /user/wjk/output/datastat
hadoop jar /home/wjk/ipcount.jar com.wjk.datastat.ip.ConvertIp -files /home/wjk/qqwry.dat,/home/wjk/dirtydata.txt  /user/hive/warehouse/active_log/dt=20120301  /user/wjk/output/datastat

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。