设为首页收藏本站
开启辅助访问

在路上

 找回密码
 立即注册
在路上 › 站点首页 ›学习 › 查看内容

嘎嘎，爬取小小编辑的动弹

2016-7-29 15:47| 发布者: zhangjf| 查看: 646| 评论: 0

摘要: 唉，压缩包居然不可以上传，一个个传上来 package MeiNvTuPian;public class DataProcessor { private static DataProcessor inst = new DataProcessor(); public PrintFile ...
唉，压缩包居然不可以上传，一个个传上来
  1. package MeiNvTuPian;
  5. public class DataProcessor {
  7. private static DataProcessor inst = new DataProcessor();
  8. public PrintFile printFile;
  9. public static DataProcessor inst(){return inst;}
  10. /**
  11. *
  12. * @param data
  13. * @param blogId
  14. */
  15. public void getBolgMsg(String data) {
  16. String str = data.substring(data.indexOf("title="原创博客"")+"title="原创博客"".length());
  17. str = str.substring(0, str.lastIndexOf("pages sm-hide")+"pages sm-hide".length());
  18. String[] blogs = str.split("title="原创博客"");
  19. for(String s : blogs){
  20. BlogModel bm = new BlogModel();
  21. BlogHtml bh = new BlogHtml();
  22. try {
  23. s = s.substring(s.indexOf("href="")+"href="".length());
  24. bm.blogUrl = s.substring(0, s.indexOf("">"));
  26. s = s.substring(s.indexOf("">")+"">".length());
  27. bm.title = s.substring(0, s.indexOf("</a>")).replaceAll(" ", "");
  29. s = s.substring(s.indexOf("time">")+"time">".length());
  30. bm.time = s.substring(0,s.indexOf("发布")).replaceAll(" ", "");
  32. this.getMoreData(bm);
  33. } catch (Exception e) {
  34. e.printStackTrace();
  35. }
  36. bh.p1 = "<p><a href=""+bm.blogUrl+"">（"+bm.time+"）"+bm.title+"</a></p>";
  37. if(bm.music == null || bm.music.equals("")){
  38. bh.p2 = "<p>本次动弹听歌，戳<a href="">（不用点了，这期没拿到）</a></p>";
  39. }else{
  40. bh.p2 = "<p>本次动弹听歌，戳<a href=""+bm.music+"">（这里）</a></p>";
  41. }
  42. bh.p3 = "<p>小树医生心理生理医务室&darr;&darr;&darr;</p>";
  43. if(bm.mmPhoto == null || bm.mmPhoto.equals("")){
  44. bh.p4 = "<p>本期弃疗</p>";
  45. }else{
  46. bh.p4 = "<p><img alt="福利" src=""+bm.mmPhoto+"" /></p>";
  47. }
  48. printFile.onData(bh);
  49. }
  50. }
  52. private void getMoreData(BlogModel bm) {
  53. String data = DataFetcher.inst().getData(bm.blogUrl);
  54. if(data.equals("")){
  55. System.out.println("动弹不存在！");
  56. return;
  57. }
  58. if(data.indexOf("手机党少年们想听歌，请使劲儿戳")>0){
  59. data = data.substring(data.indexOf("请使劲儿戳（<a href="")+"请使劲儿戳（<a href="".length());
  60. bm.music = data.substring(0,data.indexOf("""));
  61. }
  62. if(data.indexOf("小树医生心理生理医务室")>0){
  63. data = data.substring(data.indexOf("小树医生心理生理医务室")+"小树医生心理生理医务室".length());
  64. data = data.substring(data.indexOf("src="")+"src="".length());
  65. bm.mmPhoto = data.substring(0,data.indexOf("""));
  66. }
  67. }
  68. }
复制代码
  1. package MeiNvTuPian;
  3. import java.io.BufferedReader;
  4. import java.io.InputStreamReader;
  5. import java.net.URL;
  6. import java.net.URLConnection;
  8. public class DataFetcher {
  9. private static DataFetcher inst = new DataFetcher();
  11. public static DataFetcher inst() { return inst; }
  12. //模拟浏览器发出请求，并返回网页内容字符串
  13. public String getData(String urlStr)
  14. {
  15. String result = "";
  16. BufferedReader in = null;
  17. try {
  18. URL url = new URL(urlStr);
  19. System.out.println(urlStr);
  20. URLConnection connection = url.openConnection();
  21. connection.setRequestProperty("accept", "*/*");
  22. connection.setRequestProperty("connection", "Keep-Alive");
  23. connection.setRequestProperty("user-agent",
  24. "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11");
  25. connection.connect();
  26. in = new BufferedReader(new InputStreamReader(
  27. connection.getInputStream()));
  28. String line;
  29. while ((line = in.readLine()) != null) {
  30. result += line;
  31. }
  32. } catch (java.io.IOException e1)
  33. {
  34. e1.printStackTrace();
  35. }
  37. catch (Exception e) {
  38. System.out.println("get" + e);
  39. e.printStackTrace();
  40. }
  41. finally {
  42. try {
  43. if (in != null) {
  44. in.close();
  45. }
  46. } catch (Exception e2) {
  47. e2.printStackTrace();
  48. }
  49. }
  50. return result;
  51. }
  53. }
复制代码
  1. package MeiNvTuPian;
  6. public class Main {
  7. public static void main(String[] args) throws InterruptedException {
  8. int pageNum = 16;
  9. PrintFile p = new PrintFile();
  10. DataProcessor.inst().printFile = p;
  11. //p.init("blogMsg");
  12. p.init1("blogHtml");
  13. getBolgMsg(pageNum);
  14. Thread.sleep(5000);
  15. p.finish();
  16. }
  17. //处理数据
  18. private static void getBolgMsg(int pageNum) throws InterruptedException {
  19. for (int i = 1; i <= pageNum; i++)
  20. {
  21. System.out.println("page" + i);
  22. String data = DataFetcher.inst().getData("http://my.oschina.net/xxiaobian/blog?sort=time&p="+i);
  23. if(data.equals("")){
  24. break;
  25. }
  26. DataProcessor.inst().getBolgMsg(data);
  27. Thread.sleep(3000);//每获取一页数据线程睡眠3秒
  28. }
  29. }
  30. }
复制代码
  1. package MeiNvTuPian;
  3. import java.io.File;
  4. import java.io.FileWriter;
  5. import java.io.IOException;
  7. public class PrintFile {
  8. private FileWriter writer = null;
  9. //写入数据
  10. /* public void onData(BlogModel bm){
  11. try {
  12. writer.write(bm.dump()+ "n");
  13. writer.flush();
  14. } catch (IOException e) {
  15. e.printStackTrace();
  16. }
  17. }*/
  18. public void onData(BlogHtml bh){
  19. try {
  20. writer.write(bh.dump()+ "n");
  21. writer.flush();
  22. } catch (IOException e) {
  23. e.printStackTrace();
  24. }
  25. }
  26. //数据行头部写入
  27. public void init(String filename)
  28. {
  29. try {
  30. File f = new File("F:/"+filename);
  31. if (!f.exists())
  32. {
  33. f.mkdir();
  34. }
  35. writer = new FileWriter("F:/"+filename+"/"+filename+".txt");
  36. String title = "动弹url 标题 时间 歌 图片urln";
  37. writer.write(title);
  38. writer.flush();
  39. } catch (IOException e) {
  40. e.printStackTrace();
  41. }
  42. }
  43. //数据行头部写入
  44. public void init1(String filename)
  45. {
  46. try {
  47. File f = new File("F:/"+filename);
  48. if (!f.exists())
  49. {
  50. f.mkdir();
  51. }
  52. writer = new FileWriter("F:/"+filename+"/"+filename+".txt");
  53. String title = "p1 p2 p3 p4n";
  54. writer.write(title);
  55. writer.flush();
  56. } catch (IOException e) {
  57. e.printStackTrace();
  58. }
  59. }
  60. //结束
  61. public void finish()
  62. {
  63. System.out.println("done.");
  64. try {
  65. writer.close();
  66. } catch (IOException e) {
  67. e.printStackTrace();
  68. }
  69. }
  70. }
复制代码
  1. package MeiNvTuPian;
  3. public class BlogHtml {
  4. String p1;
  5. String p2;
  6. String p3;
  7. String p4;
  8. public String dump(){
  9. String seperator = " ";
  10. String ret = p1 + seperator
  11. + p2 + seperator
  12. + p3 + seperator
  13. + p4;
  14. return ret;
  15. }
  16. }
复制代码
  1. package MeiNvTuPian;
  3. /**
  4. *
  5. * @author Administrator
  6. * 对象类
  7. */
  8. public class BlogModel {
  9. String blogUrl;
  10. String title;
  11. String time;
  12. String music;
  13. String mmPhoto;
  15. public String dump(){
  16. String seperator = " ";
  17. String ret = blogUrl + seperator
  18. + title + seperator
  19. + time + seperator
  20. + music + seperator
  21. + mmPhoto;
  22. return ret;
  23. }
  24. }
复制代码
收藏 邀请
上一篇：仿js的url encodeURI方法下一篇：批量查询快递单号

最新评论

相关分类

小黑屋|在路上 ( 蜀ICP备15035742号-1 )  

;

GMT+8, 2025-5-6 15:42

Copyright 2015-2025 djqfx

返回顶部