load->library(array('pagination', 'querystring'));
}
/**
* 목록을 가져오는 메소드입니다
*/
public function index()
{
$this->accesslevel->check_admin('read');
$view = array();
$view['view'] = array();
/**
* 페이지에 숫자가 아닌 문자가 입력되거나 1보다 작은 숫자가 입력되면 에러 페이지를 보여줍니다.
*/
$param =& $this->querystring;
$page = (((int) $this->input->get('page')) > 0) ? ((int) $this->input->get('page')) : 1;
$findex = $this->input->get('findex') ? $this->input->get('findex') : $this->{$this->modelname}->primary_key;
$forder = $this->input->get('forder', null, 'desc');
$sfield = $this->input->get('sfield', null, '');
$skeyword = $this->input->get('skeyword', null, '');
$per_page = admin_listnum();
$offset = ($page - 1) * $per_page;
/**
* 게시판 목록에 필요한 정보를 가져옵니다.
*/
$this->{$this->modelname}->allow_search_field = array('app_id', 'crawl_title'); // 검색이 가능한 필드
$this->{$this->modelname}->search_field_equal = array('app_id'); // 검색중 like 가 아닌 = 검색을 하는 필드
$this->{$this->modelname}->allow_order_field = array('crawl_id'); // 정렬이 가능한 필드
$where = array();
if ($this->input->get('app_id')) {
$where['app_id'] = $this->input->get('app_id');
}
$result = $this->{$this->modelname}
->get_admin_list($per_page, $offset, $where, '', $findex, $forder, $sfield, $skeyword);
$list_num = $result['total_rows'] - ($page - 1) * $per_page;
if (element('list', $result)) {
foreach (element('list', $result) as $key => $val) {
$result['list'][$key]['num'] = $list_num--;
if (element('cat_id', $val)) {
$result['list'][$key]['category'] = $this->Contents_category_model->get_one(element('cat_id', $val));
$ccwhere = array(
'crawl_id' => element('crawl_id', $val),
);
$result['list'][$key]['crawled_count'] = $this->Contents_model->count_by($ccwhere);
}
}
}
$view['view']['data'] = $result;
$view['view']['applist'] = $this->Apps_model->app_list();
/**
* primary key 정보를 저장합니다
*/
$view['view']['primary_key'] = $this->{$this->modelname}->primary_key;
/**
* 페이지네이션을 생성합니다
*/
$config['base_url'] = admin_url($this->pagedir) . '?' . $param->replace('page');
$config['total_rows'] = $result['total_rows'];
$config['per_page'] = $per_page;
$this->pagination->initialize($config);
$view['view']['paging'] = $this->pagination->create_links();
$view['view']['page'] = $page;
/**
* 쓰기 주소, 삭제 주소등 필요한 주소를 구합니다
*/
$search_option = array('crawl_title' => '제목');
$view['view']['skeyword'] = ($sfield && array_key_exists($sfield, $search_option)) ? $skeyword : '';
$view['view']['search_option'] = search_option($search_option, $sfield);
$view['view']['listall_url'] = admin_url($this->pagedir);
$view['view']['write_url'] = admin_url($this->pagedir . '/write');
$view['view']['list_delete_url'] = admin_url($this->pagedir . '/listdelete/?' . $param->output());
/**
* 어드민 레이아웃을 정의합니다
*/
$layoutconfig = array('layout' => 'layout', 'skin' => 'index');
$view['layout'] = $this->layoutlib->admin($layoutconfig, $this->configlib->get_device_view_type());
$this->data = $view;
$this->layout = element('layout_skin_file', element('layout', $view));
$this->view = element('view_skin_file', element('layout', $view));
}
/**
* 게시판 글쓰기 또는 수정 페이지를 가져오는 메소드입니다
*/
public function write($pid = 0)
{
$this->accesslevel->check_admin('read');
$view = array();
$view['view'] = array();
/**
* 프라이머리키에 숫자형이 입력되지 않으면 에러처리합니다
*/
if ($pid) {
$pid = (int) $pid;
if (empty($pid) OR $pid < 1) {
show_404();
}
}
$primary_key = $this->{$this->modelname}->primary_key;
/**
* 수정 페이지일 경우 기존 데이터를 가져옵니다
*/
$getdata = array();
if ($pid) {
$getdata = $this->{$this->modelname}->get_one($pid);
} else if ($this->input->get('app_id')) {
$getdata['app_id'] = $this->input->get('app_id');
}
/**
* Validation 라이브러리를 가져옵니다
*/
$this->load->library('form_validation');
$view['view']['upload_file_count'] = $upload_file_count = 0;
/**
* 전송된 데이터의 유효성을 체크합니다
*/
$config = array(
array('field' => 'app_id', 'label' => '어플 이름', 'rules' => 'trim|required'),
array('field' => 'cat_id', 'label' => '카테고리', 'rules' => 'trim|required'),
array('field' => 'crawl_title', 'label' => '제목', 'rules' => 'trim|required'),
array('field' => 'site_type', 'label' => '사이트 종류', 'rules' => 'trim|required'),
array('field' => 'site_key', 'label' => '사이트 키', 'rules' => 'trim|required'),
array('field' => 'crawl_category', 'label' => '크롤할 카테고리', 'rules' => 'trim'),
array('field' => 'crawl_category_name', 'label' => '크롤할 카테고리', 'rules' => 'trim'),
array('field' => 'start_page', 'label' => '시작페이지', 'rules' => 'trim'),
array('field' => 'end_page', 'label' => '마지막 페이지', 'rules' => 'trim'),
);
$this->form_validation->set_rules($config);
$form_validation = $this->form_validation->run();
$file_error = '';
$uploadfiledata = '';
$uploadfiledata2 = '';
if ($upload_file_count > 0 && $form_validation) {
$this->load->library('upload');
if (isset($_FILES) && isset($_FILES['upload_file']) && isset($_FILES['upload_file']['name']) && is_array($_FILES['upload_file']['name'])) {
$filecount = count($_FILES['upload_file']['name']);
$upload_path = config_item('uploads_dir') . '/files/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
$upload_path .= cdate('Y') . '/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
$upload_path .= cdate('m') . '/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
foreach ($_FILES['upload_file']['name'] as $i => $value) {
if ($value) {
$uploadconfig = '';
$uploadconfig['upload_path'] = $upload_path;
$uploadconfig['allowed_types'] = '*';
$uploadconfig['max_size'] = 10240;
$uploadconfig['encrypt_name'] = true;
$this->upload->initialize($uploadconfig);
$_FILES['userfile']['name'] = $_FILES['upload_file']['name'][$i];
$_FILES['userfile']['type'] = $_FILES['upload_file']['type'][$i];
$_FILES['userfile']['tmp_name'] = $_FILES['upload_file']['tmp_name'][$i];
$_FILES['userfile']['error'] = $_FILES['upload_file']['error'][$i];
$_FILES['userfile']['size'] = $_FILES['upload_file']['size'][$i];
if ($this->upload->do_upload()) {
$filedata = $this->upload->data();
$uploadfiledata[$i]['ufi_filename'] = cdate('Y') . '/' . cdate('m') . '/' . element('file_name', $filedata);
$uploadfiledata[$i]['ufi_originname'] = element('orig_name', $filedata);
$uploadfiledata[$i]['ufi_filesize'] = intval(element('file_size', $filedata) * 1024);
$uploadfiledata[$i]['ufi_width'] = element('image_width', $filedata) ? element('image_width', $filedata) : 0;
$uploadfiledata[$i]['ufi_height'] = element('image_height', $filedata) ? element('image_height', $filedata) : 0;
$uploadfiledata[$i]['ufi_type'] = str_replace('.', '', element('file_ext', $filedata));
$uploadfiledata[$i]['is_image'] = element('is_image', $filedata) ? element('is_image', $filedata) : 0;
} else {
$file_error = $this->upload->display_errors();
break;
}
}
}
}
if (isset($_FILES) && isset($_FILES['upload_file_update'])
&& isset($_FILES['upload_file_update']['name'])
&& is_array($_FILES['upload_file_update']['name'])
&& $file_error === '') {
$filecount = count($_FILES['upload_file_update']['name']);
$upload_path = config_item('uploads_dir') . '/files/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
$upload_path .= cdate('Y') . '/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
$upload_path .= cdate('m') . '/';
if (is_dir($upload_path) === false) {
mkdir($upload_path, 0707);
$file = $upload_path . 'index.php';
$f = @fopen($file, 'w');
@fwrite($f, '');
@fclose($f);
@chmod($file, 0644);
}
foreach ($_FILES['upload_file_update']['name'] as $i => $value) {
if ($value) {
$uploadconfig = '';
$uploadconfig['upload_path'] = $upload_path;
$uploadconfig['allowed_types'] = '*';
$uploadconfig['max_size'] = 10240;
$uploadconfig['encrypt_name'] = true;
$this->upload->initialize($uploadconfig);
$_FILES['userfile']['name'] = $_FILES['upload_file_update']['name'][$i];
$_FILES['userfile']['type'] = $_FILES['upload_file_update']['type'][$i];
$_FILES['userfile']['tmp_name'] = $_FILES['upload_file_update']['tmp_name'][$i];
$_FILES['userfile']['error'] = $_FILES['upload_file_update']['error'][$i];
$_FILES['userfile']['size'] = $_FILES['upload_file_update']['size'][$i];
if ($this->upload->do_upload()) {
$filedata = $this->upload->data();
$olduploadfile = $this->Upload_file_model->get_one($i);
if ((int) element('origin_id', $olduploadfile) !== (int) element('crawl_id', $getdata)) {
alert('잘못된 접근입니다');
}
@unlink(config_item('uploads_dir') . '/files/' . element('ufi_filename', $oldpostfile));
$uploadfiledata2[$i]['ufi_id'] = $i;
$uploadfiledata2[$i]['ufi_filename'] = cdate('Y') . '/' . cdate('m') . '/' . element('file_name', $filedata);
$uploadfiledata2[$i]['ufi_originname'] = element('orig_name', $filedata);
$uploadfiledata2[$i]['ufi_filesize'] = intval(element('file_size', $filedata) * 1024);
$uploadfiledata2[$i]['ufi_width'] = element('image_width', $filedata)
? element('image_width', $filedata) : 0;
$uploadfiledata2[$i]['ufi_height'] = element('image_height', $filedata)
? element('image_height', $filedata) : 0;
$uploadfiledata2[$i]['ufi_type'] = str_replace('.', '', element('file_ext', $filedata));
$uploadfiledata2[$i]['is_image'] = element('is_image', $filedata)
? element('is_image', $filedata) : 0;
} else {
$file_error = $this->upload->display_errors();
break;
}
}
}
}
}
/**
* 유효성 검사를 하지 않는 경우, 또는 유효성 검사에 실패한 경우입니다.
* 즉 글쓰기나 수정 페이지를 보고 있는 경우입니다
*/
if ($form_validation === false OR $file_error !== '') {
$view['view']['applist'] = $applist = $this->Apps_model->app_list();
if ( ! $pid) {
if ($this->input->get('app_id')) {
$getdata['app_id'] = $app_id = $this->input->get('app_id');
$getdata['app_name'] = html_escape(element('app_name', element($app_id, $applist)));
} else {
alert('잘못된 접근입니다');
}
} else {
$app_id = element('app_id', $getdata);
$getdata['app_name'] = html_escape(element('app_name', element($app_id, $applist)));
}
$view['view']['catlist'] = $catlist = $this->Contents_category_model->cat_list($app_id);
$where = array(
'cat_parent' => '0',
'app_id' => $app_id,
);
$getdata['cat'] = $this->Contents_category_model->get('', '', $where, '', '', 'cat_sort', 'desc');
$view['view']['data'] = $getdata;
/**
* primary key 정보를 저장합니다
*/
$view['view']['primary_key'] = $primary_key;
if (element('crawl_id', $getdata)) {
$uploadwhere = array(
'origin_table' => 'crawl_list',
'origin_id' => element('crawl_id', $getdata),
);
$view['view']['file'] = $file = $this->Upload_file_model->get('', '', $uploadwhere, '', '', 'ufi_id', 'ASC');
if ($file && is_array($file)) {
foreach ($file as $key => $value) {
$view['view']['file'][$key]['download_link']
= site_url('download/downfiles/' . element('ufi_id', $value));
}
}
}
/**
* 어드민 레이아웃을 정의합니다
*/
$layoutconfig = array('layout' => 'layout', 'skin' => 'write');
$view['layout'] = $this->layoutlib->admin($layoutconfig, $this->configlib->get_device_view_type());
$this->data = $view;
$this->layout = element('layout_skin_file', element('layout', $view));
$this->view = element('view_skin_file', element('layout', $view));
} else {
/**
* 유효성 검사를 통과한 경우입니다.
* 즉 데이터의 insert 나 update 의 process 처리가 필요한 상황입니다
*/
$updatedata = array(
'app_id' => $this->input->post('app_id', null, ''),
'cat_id' => $this->input->post('cat_id', null, ''),
'crawl_title' => $this->input->post('crawl_title', null, ''),
'site_type' => $this->input->post('site_type', null, ''),
'site_key' => $this->input->post('site_key', null, ''),
'crawl_category' => $this->input->post('crawl_category', null, ''),
'crawl_category_name' => $this->input->post('crawl_category_name', null, ''),
'start_page' => $this->input->post('start_page', null, ''),
'end_page' => $this->input->post('end_page', null, ''),
);
/**
* 게시물을 수정하는 경우입니다
*/
if ($this->input->post($primary_key)) {
$this->accesslevel->check_admin('edit');
$updatedata['updated_at'] = cdate('Y-m-d H:i:s');
$this->{$this->modelname}->update($this->input->post($primary_key), $updatedata);
$origin_id = $this->input->post($primary_key);
$this->session->set_flashdata('message','정상적으로 수정되었습니다');
} else {
$this->accesslevel->check_admin('write');
/**
* 게시물을 새로 입력하는 경우입니다
*/
$updatedata['created_at'] = cdate('Y-m-d H:i:s');
$updatedata['updated_at'] = cdate('Y-m-d H:i:s');
$origin_id = $this->{$this->modelname}->insert($updatedata);
$this->session->set_flashdata('message','정상적으로 입력되었습니다');
}
$file_updated = false;
$file_changed = false;
if (is_array($uploadfiledata) && count($uploadfiledata) > 0) {
foreach ($uploadfiledata as $pkey => $pval) {
if ($pval) {
$fileupdate = array(
'origin_id' => $origin_id,
'origin_table' => 'crawl_list',
'user_id' => $this->userlib->item('user_id'),
'ufi_originname' => element('ufi_originname', $pval),
'ufi_filename' => element('ufi_filename', $pval),
'ufi_filesize' => element('ufi_filesize', $pval),
'ufi_width' => element('ufi_width', $pval),
'ufi_height' => element('ufi_height', $pval),
'ufi_type' => element('ufi_type', $pval),
'ufi_is_image' => element('is_image', $pval),
'ufi_datetime' => cdate('Y-m-d H:i:s'),
'ufi_ip' => $this->input->ip_address(),
);
$file_id = $this->Upload_file_model->insert($fileupdate);
$file_updated = true;
}
}
$file_changed = true;
}
if (is_array($uploadfiledata2) && count($uploadfiledata2) > 0) {
foreach ($uploadfiledata2 as $pkey => $pval) {
if ($pval) {
$fileupdate = array(
'user_id' => $this->userlib->item('user_id'),
'ufi_originname' => element('ufi_originname', $pval),
'ufi_filename' => element('ufi_filename', $pval),
'ufi_filesize' => element('ufi_filesize', $pval),
'ufi_width' => element('ufi_width', $pval),
'ufi_height' => element('ufi_height', $pval),
'ufi_type' => element('ufi_type', $pval),
'ufi_is_image' => element('is_image', $pval),
'ufi_datetime' => cdate('Y-m-d H:i:s'),
'ufi_ip' => $this->input->ip_address(),
);
$this->Upload_file_model->update($pkey, $fileupdate);
$file_changed = true;
}
}
}
if ($this->input->post('upload_file_del')) {
foreach ($this->input->post('upload_file_del') as $key => $val) {
if ($val === '1' && ! isset($uploadfiledata2[$key])) {
$olduploadfile = $this->Upload_file_model->get_one($key);
if ( ! element('origin_id', $olduploadfile) OR (int) element('origin_id', $olduploadfile) !== (int) $origin_id) {
alert('잘못된 접근입니다.');
}
@unlink(config_item('uploads_dir') . '/files/' . element('ufi_filename', $olduploadfile));
$this->Upload_file_model->delete($key);
$file_changed = true;
}
}
}
/**
* 게시물의 신규입력 또는 수정작업이 끝난 후 목록 페이지로 이동합니다
*/
$param =& $this->querystring;
$redirecturl = admin_url($this->pagedir . '?' . $param->output());
redirect($redirecturl);
}
}
public function crawler($pid = 0)
{
set_time_limit(0);
$this->db->query('SET session wait_timeout=28800');
//simple html dom 셋팅
include_once FCPATH . 'plugin/simplehtmldom/simple_html_dom.php';
$crawl = $this->Crawl_list_model->get_one($pid);
$historyinsert = array(
'crawl_id' => element('crawl_id', $crawl),
'app_id' => element('app_id', $crawl),
'cat_id' => element('cat_id', $crawl),
'site_type' => element('site_type', $crawl),
'site_key' => element('site_key', $crawl),
'history_at' => cdate('Y-m-d H:i:s'),
);
$history_id = $this->Crawl_history_model->insert($historyinsert);
$crawl['history_id'] = $history_id;
if (element('site_type', $crawl) == '1') { // 네이버 블로그
$this->_naverblog($crawl);
}
if (element('site_type', $crawl) == '2') { // 네이버 포스트
$this->_naverpost($crawl);
}
if (element('site_type', $crawl) == '3') { // 빙글
$this->_vingle($crawl);
}
if (element('site_type', $crawl) == '4') { // 카카오 스토리
$this->_kakaostory($crawl);
}
$updatedata = array(
'latest_crawled_at' => cdate('Y-m-d H:i:s'),
);
$this->Crawl_list_model->update($pid, $updatedata);
echo '수집 완료!!
';
}
public function _naverblog($crawl) {
$crawling_url = array();
$category_id = element('crawl_category', $crawl) ? element('crawl_category', $crawl) : 0;
$list_url = 'http://m.blog.naver.com/rego/PostListInfo.nhn?blogId=' . element('site_key', $crawl) . '&categoryNo=' . $category_id . '¤tPage=1&logCode=0';
$opt = array(
'url' => $list_url,
'followlocation' => true,
'referer' => 'http://m.blog.naver.com/' . element('site_key', $crawl) . '?currentPage=1',
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$result = curl_data($opt);
$result = substr($result, 5);
//echo $result;
$array = json_decode($result, true);
$last_page = element('totalPage', element('result', $array));
//echo $last_page;
$start_page = element('start_page', $crawl) ? element('start_page', $crawl) : 1;
$end_page = element('end_page', $crawl) ? element('end_page', $crawl) : $last_page;
for ($page = $start_page; $page<= $end_page; $page++) {
$list_url = 'http://m.blog.naver.com/rego/PostListInfo.nhn?blogId=' . element('site_key', $crawl) . '&categoryNo=' . $category_id . '¤tPage=' . $page . '&logCode=0';
$opt = array(
'url' => $list_url,
'followlocation' => true,
'referer' => 'http://m.blog.naver.com/' . element('site_key', $crawl) . '?currentPage=' . $page,
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$result = curl_data($opt);
$result = substr($result, 5);
//echo $result;
$array = json_decode($result, true);
if (element('postViewList', element('result', $array))) {
foreach (element('postViewList', element('result', $array)) as $key => $value) {
$crawling_url[] = 'http://m.blog.naver.com/' . element('blogId', $value) . '/' . element('logNo', $value);
}
}
//텀을 두기
usleep(300000); //1000000 = 1SEC
}
if ($crawling_url) {
$max_sort = $this->Contents_model->select_max_sort(element('app_id', $crawl));
foreach ($crawling_url as $key => $value) {
$where = array(
'app_id' => element('app_id', $crawl),
'con_origin_url' => $value,
);
$content = $this->Contents_model->get_one('', '', $where);
if (element('con_id', $content)) continue;
$opt = array(
'url' => $value,
'followlocation' => true,
'referer' => 'http://m.blog.naver.com/' . element('site_key', $crawl),
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$html = curl_data($opt);
$dom = str_get_html($html);
$con_title = '';
if($dom->find("title")) {
$con_title = $dom->find("title", 0)->plaintext;
$con_title = str_replace(': 네이버 블로그', '', $con_title);
} else {
continue;
}
$con_content = '';
if($dom->find("#viewTypeSelector")) {
$con_content = $dom->find("#viewTypeSelector", 0)->innertext;
} else {
continue;
}
$con_origin_created_at = null;
if($dom->find(".num")) {
$num_text = explode('.', $dom->find(".num", 0)->innertext);
$con_origin_created_at = trim($num_text[0]) . ':' . trim($num_text[1]) . ':' . trim($num_text[2]) . ' ' . trim($num_text[3]) . ':00';
}
$max_sort++;
$insertdata = array(
'app_id' => element('app_id', $crawl),
'con_sort' => $max_sort,
'cat1_id' => element('cat_id', $crawl),
'user_id' => '1',
'user_name' => '관리자',
'con_title' => $con_title,
'con_content' => $con_content,
'crawl_id' => element('crawl_id', $crawl),
'history_id' => element('history_id', $crawl),
'con_origin_url' => $value,
'con_hit' => '0',
'con_like' => '0',
'con_created_at' => cdate('Y-m-d H:i:s'),
'con_updated_at' => cdate('Y-m-d H:i:s'),
'con_origin_created_at' => $con_origin_created_at,
'is_open' => '0',
'is_del' => '0',
);
$insert_id = $this->Contents_model->insert($insertdata);
if($dom->find(".post_tag li")) {
foreach ($dom->find(".post_tag li") as $e) {
$tag = str_replace('#', '', strip_tags($e->innertext));
if ($tag) {
$tagdata = array(
'con_id' => $insert_id,
'app_id' => element('app_id', $crawl),
'tag_name' => $tag,
);
$this->Contents_tag_model->insert($tagdata);
}
}
}
//memory control
@$dom->clear();
//echo $insert_id." / ";
//flush();
//텀을 두기
usleep(100000); //1000000 = 1SEC
}
}
}
public function _naverpost($crawl) {
$crawling_url = array();
$category_id = element('crawl_category', $crawl) ? element('crawl_category', $crawl) : 0;
$list_url = 'http://m.post.naver.com/async/my.nhn?memberNo=' . element('site_key', $crawl) . '&postListViewType=0&isExpertMy=true';
$opt = array(
'url' => $list_url,
'followlocation' => true,
'referer' => 'http://m.post.naver.com/my.nhn?memberNo=' . element('site_key', $crawl),
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$result = curl_data($opt);
//echo $result;
$result = str_replace("\'", "'", $result);
$array = json_decode($result, true);
//print_r2($array);
$list_count = element('listCount', $array);
$total_count = element('totalCount', $array);
$last_page = ceil($total_count / $list_count);
$start_page = element('start_page', $crawl) ? element('start_page', $crawl) : 1;
$end_page = element('end_page', $crawl) ? element('end_page', $crawl) : $last_page;
for ($page = $start_page; $page<= $end_page; $page++) {
$list_url = 'http://m.post.naver.com/async/my.nhn?memberNo=' . element('site_key', $crawl) . '&postListViewType=0&isExpertMy=true&fromNo=' . $page . '&totalCount=' . $total_count;
$opt = array(
'url' => $list_url,
'followlocation' => true,
'referer' => 'http://m.post.naver.com/my.nhn?memberNo=' . element('site_key', $crawl),
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$result = curl_data($opt);
$result = str_replace("\'", "'", $result);
$array = json_decode($result, true);
$html = stripslashes(element('html', $array));
$dom = str_get_html($html);
if($dom->find("button.btn_share")) {
foreach($dom->find("button.btn_share") as $e) {
if ($e->{'data-url'}) {
$crawling_url[] = $e->{'data-url'};
}
}
} else {
//break;
}
//memory control
@$dom->clear();
//텀을 두기
usleep(300000); //1000000 = 1SEC
}
if ($crawling_url) {
$max_sort = $this->Contents_model->select_max_sort(element('app_id', $crawl));
foreach ($crawling_url as $key => $value) {
$where = array(
'app_id' => element('app_id', $crawl),
'con_origin_url' => $value,
);
$content = $this->Contents_model->get_one('', '', $where);
if (element('con_id', $content)) continue;
$opt = array(
'url' => $value,
'followlocation' => true,
'referer' => 'http://m.post.naver.com/my.nhn?memberNo=' . element('site_key', $crawl),
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$html = curl_data($opt);
$dom = str_get_html($html);
$con_title = '';
if($dom->find("title")) {
$con_title = $dom->find("title", 0)->plaintext;
$con_title = str_replace(': 네이버 포스트', '', $con_title);
} else {
continue;
}
$con_content = '';
if($dom->find("#__clipContent")) {
$con_content = $dom->find("#__clipContent", 0)->innertext;
} else {
continue;
}
$con_origin_created_at = null;
if($dom->find(".se_publishDate")) {
$con_origin_created_at = date('Y-m-d H:i:s', strtotime($dom->find(".se_publishDate", 0)->innertext));
}
$max_sort++;
$insertdata = array(
'app_id' => element('app_id', $crawl),
'con_sort' => $max_sort,
'cat1_id' => element('cat_id', $crawl),
'user_id' => '1',
'user_name' => '관리자',
'con_title' => $con_title,
'con_content' => $con_content,
'crawl_id' => element('crawl_id', $crawl),
'history_id' => element('history_id', $crawl),
'con_origin_url' => $value,
'con_hit' => '0',
'con_like' => '0',
'con_created_at' => cdate('Y-m-d H:i:s'),
'con_updated_at' => cdate('Y-m-d H:i:s'),
'con_origin_created_at' => $con_origin_created_at,
'is_open' => '0',
'is_del' => '0',
);
$insert_id = $this->Contents_model->insert($insertdata);
if($dom->find(".tag_area a")) {
foreach ($dom->find(".tag_area a") as $e) {
$tag = str_replace('#', '', strip_tags($e->innertext));
if ($tag) {
$tagdata = array(
'con_id' => $insert_id,
'app_id' => element('app_id', $crawl),
'tag_name' => $tag,
);
$this->Contents_tag_model->insert($tagdata);
}
}
}
//memory control
@$dom->clear();
//텀을 두기
usleep(100000); //1000000 = 1SEC
}
}
}
public function _naverband($crawl) {
$crawling_url = array();
$list_url = 'https://api.band.us/v1.5.3/get_posts?client_info=%7B%22language%22%3A%22ko%22%2C%22country%22%3A%22KR%22%2C%22version%22%3A1%2C%22agent_type%22%3A%22web%22%2C%22agent_version%22%3A%223.3.1%22%2C%22resolution_type%22%3A100%7D&language=ko&country=KR&version=1&akey=bbc59b0b5f7a1c6efe950f6236ccda35&DEVICE-TIME-ZONE-ID=Asia%2FSeoul&DEVICE-TIME-ZONE-MS-OFFSET=32400000&md=2xm8Zc%2BS5FNm2P0CEKTXV0gQa78X2Z19r8UvqVtG47k%3D&ts=1496885711478&band_no=' . element('site_key', $crawl) . '&post_no=&direction=before&with_notice=true&count=100&resolution_type=4&_=1496885711329';
$opt = array(
'url' => $list_url,
'followlocation' => true,
'referer' => 'http://band.us/',
'useragent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
);
$result = curl_data($opt);
$array = json_decode($result, true);
//print_r2($array);
$max_sort = $this->Contents_model->select_max_sort(element('app_id', $crawl));
if (element('posts', element('result_data', $array))) {
foreach (element('posts', element('result_data', $array)) as $value) {
$post_url = 'http://band.us/#!/band/' . element('band_no', $value) . '/post/' . element('post_no', $value);
$where = array(
'app_id' => element('app_id', $crawl),
'con_origin_url' => $post_url,
);
$content = $this->Contents_model->get_one('', '', $where);
if (element('con_id', $content)) continue;
$max_sort++;
$con_title = element('name', element('author', $value));
$con_content = '';
if (element('photos', $value)) {
foreach (element('photos', $value) as $photo) {
$con_content .= '';
$con_content .= '
 . ')
';
$con_content .= '
';
$con_content .= '
 . ')
';
$con_content .= '
';
}
if (element('type', $cval) == 'hashtag') {
$tags[] = str_replace('#', '', element('text', $cval));
}
}
}
}
}
*/
$con_content .= element('content', $value);
$insertdata = array(
'app_id' => element('app_id', $crawl),
'con_sort' => $max_sort,
'cat1_id' => element('cat_id', $crawl),
'user_id' => '1',
'user_name' => '관리자',
'con_title' => $con_title,
'con_content' => $con_content,
'crawl_id' => element('crawl_id', $crawl),
'history_id' => element('history_id', $crawl),
'con_origin_url' => $post_url,
'con_hit' => '0',
'con_like' => '0',
'con_created_at' => cdate('Y-m-d H:i:s'),
'con_updated_at' => cdate('Y-m-d H:i:s'),
'con_origin_created_at' => element('created_at', $value),
'is_open' => '0',
'is_del' => '0',
);
$insert_id = $this->Contents_model->insert($insertdata);
/*
if ($tags) {
foreach ($tags as $tag) {
$tagdata = array(
'con_id' => $insert_id,
'app_id' => element('app_id', $crawl),
'tag_name' => $tag,
);
$this->Contents_tag_model->insert($tagdata);
}
}
*/
//echo $insert_id." / ";
//flush();
//텀을 두기
usleep(100000); //1000000 = 1SEC
}
}
}
/**
* 목록 페이지에서 선택삭제를 하는 경우 실행되는 메소드입니다
*/
public function listdelete()
{
$this->accesslevel->check_admin('delete');
/**
* 체크한 게시물의 삭제를 실행합니다
*/
if ($this->input->post('chk') && is_array($this->input->post('chk'))) {
foreach ($this->input->post('chk') as $val) {
if ($val) {
$this->{$this->modelname}->delete($val);
}
}
}
/**
* 삭제가 끝난 후 목록페이지로 이동합니다
*/
$this->session->set_flashdata('message','정상적으로 삭제되었습니다');
$param =& $this->querystring;
$redirecturl = admin_url($this->pagedir . '?' . $param->output());
redirect($redirecturl);
}
}