1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280

#This 檔案 uses UTF-8 ; coding=utf-8 ;;; by tsaiwn@cs.nctu.edu.tw
# For original Dummy_Device, Google Search github + dummy + iottalk  for other files
import time, DAN, requests, random 
import threading, sys # for using a Thread to read keyboard INPUT

# ServerURL = 'http://Your_server_IP_or_DomainName:9999' #with no secure connection
# ServerURL = 'http://192.168.20.101:9999' #with no secure connection
#  注意你用的 IoTtalk 伺服器網址或 IP  
ServerURL = 'https://demo.iottalk.tw'  # with SSL secure connection
# ServerURL = 'https://Your_DomainName' #with SSL connection  (IP can not be used with https)
Reg_addr = None #if None, Reg_addr = MAC address #(本來在 DAN.py 要這樣做 :-) 
# Note that Reg_addr 在以下三句會被換掉! # the mac_addr in DAN.py is NOT used
mac_addr = 'CD8600D38' + str( random.randint(100,999 ) )  # put here for easy to modify :-) 
# 若希望每次執行這程式都被認為同一個 Dummy_Device, 要把上列 mac_addr 寫死, 不要用亂數。
Reg_addr = mac_addr  # Note that the mac_addr generated in DAN.py always be the same cause using UUID ! 

DAN.profile['dm_name']='Dummy_Device'   # you can change this but should also add the DM in server
DAN.profile['df_list']=['Dummy_Sensor', 'Dummy_Control']   # Check IoTtalk to see what IDF/ODF the DM has
DAN.profile['d_name']= "TWN_D."+ str( random.randint(100,999 ) ) +"_"+ DAN.profile['dm_name'] # None
DAN.device_registration_with_retry(ServerURL, Reg_addr) 
print("dm_name is ", DAN.profile['dm_name']) ; print("Server is ", ServerURL);
# global gotInput, theInput, allDead    ## 主程式不必宣告 globel, 但寫了也 OK
gotInput=False
theInput="haha"
allDead=False

from bs4 import BeautifulSoup
from selenium import webdriver
import datetime

def clearLists( ):
  global date,temp,weather, wind_direction, wind_speed, gust_wind
  global visible, hum, pre, rain, sunlight
  date = []
  temp = []
  weather = []
  wind_direction = []
  wind_speed = []
  gust_wind = []
  visible = []
  hum = []
  pre = []
  rain = []
  sunlight = []

driverReady = False
region = 'hsinchu'
reginTitle="新竹測站觀測資料"

url = 'https://www.cwb.gov.tw/V8/C/W/OBS_Station.html?ID=46757'
from time import sleep   # so that we do NOT have to write time.sleep( ) :-) 
def initCWB(): 
   global  driver, driverReady
   #啟動模擬瀏覽器
   from selenium.webdriver.chrome.options import Options
   options = Options()
   options.add_argument('--no-sandbox') # Bypass OS security model
   options.add_argument("--log-level=3");  # "--disable-logging"  # 尚可, 仍有 Devtool 信息
   ## options.add_argument("--silent")  #  沒用 ! ! !
## get rid off Devtool msg / chrome msg ... 
   options.add_experimental_option('excludeSwitches', ['enable-logging'])  # 要! 要 !!要 !!!
## do NOT open the browser
   options.add_argument("--headless")
##
   try:
      driver = webdriver.Chrome(options=options) # for Windows: 把 chromedriver.exe 放要執行 .py 程式的目錄即可 !
      driverReady = True
   except:
      print("!!! Can NOT get WEB driver !!!")
   #取得網頁代碼
   try:
     driver.get(url)
     open(region+'.html','wb').write(driver.page_source.encode('utf-8'))
   except:
     print("!!! get URL Fail :" + url)

# #對list中的每一項 <tr>
#for tr in trs:
def processTr(tr):
  global date,temp,weather, wind_direction, wind_speed, gust_wind
  global visible, hum, pre, rain, sunlight
  # print("===NOW ", datetime.datetime.now() , flush=True) 
  if tr != None:
    # 使用datetime取得時間年分
    year = str(datetime.datetime.now().year)
#   取時間, <tr>內的<th>, <th>內為時間 月/日<br>時:分
    d = tr.th.text
    d = year + d
#   字串轉為datetime格式
    date.append(datetime.datetime.strptime(d, '%Y%m/%d %H:%M'))
    temp.append(tr.find('td',{'headers':'temp'}).text)
    ggyy = tr.find('td',{'headers':'weather'}).find('img')
    if ggyy != None:
       weather.append(ggyy['title'])
    else: weather.append( "不知道" )
    wind_direction.append(tr.find('td',{'headers':'w-1'}).text)
    wind_speed.append(tr.find('td',{'headers':'w-2'}).text)
    gust_wind.append(tr.find('td',{'headers':'w-3'}).text)
    visible.append(tr.find('td',{'headers':'visible-1'}).text)
    hum.append(tr.find('td',{'headers':'hum'}).text)
    pre.append(tr.find('td',{'headers':'pre'}).text)
    rain.append(tr.find('td',{'headers':'rain'}).text)
    sunlight.append(tr.find('td',{'headers':'sunlight'}).text)

def buildTable( ):
  global table    ## 以下兩列的  global  其實不必要 !
  global date,temp,weather, wind_direction, wind_speed, gust_wind
  global visible, hum, pre, rain, sunlight
  table = {
"觀測時間":date,
"溫度(°C)":temp,
"天氣":weather,
"風向":wind_direction,
"風力 (m/s)":wind_speed,
"陣風 (m/s)":gust_wind,
"能見度(公里)":visible,
"相對溼度(%)":hum,
"海平面氣壓(百帕)":pre,
"當日累積雨量(毫米)":rain,
"日照時數(小時)":sunlight
}

## 這函數會把 line 和 line2 準備好, 分別是次新和最新 的氣象資料 
def prepareData( ):
    global  driver, line, line2, driverReady, table
    #指定 lxml 作為解析器
    soup = BeautifulSoup(driver.page_source, features='lxml')
    sleep(1)  # 其實不必睡一下 :-)
    # <tbody id='obstime'>  # 抓過去24小時資料 (氣象局網頁給的) 
    tbody = soup.find('tbody',{'id':'obstime'})

    # <tbody>内所有<tr>標籤
    trs = tbody.find_all('tr')   ## Google 搜尋 BeautifulSoup + find
    # 把次新的(trs[1]) 串成一列 放 line 
    tr=trs[1]    ## 次新的氣象資料 
    clearLists( )   # 清除所有 11 項 List
    processTr(tr) # 把 tr 資料抓出塞入 11 個 List
    buildTable( ) # 根據 11 個 List 建立 table 這 { dictionary }
    line ="\r\n"
    for gg  in table:     # table 內共有 11 項 
       yy = table[gg][0]   # 每項都是只一個元素的  [List] 
       if gg == "觀測時間":  # 這項特別處理一下
          yy = yy.strftime("%Y-%b-%d %H:%M:%S")
       line += gg +":" + yy
       line += "\r\n"
    #print("line: ", line, flush=True)   
    tr=trs[0]   #  最新的; 因只要 push 兩列, 偷懶沒寫函數, 複製上面一段過來用 :-)
    clearLists( )   # 清除所有 11 項 List   # (主要是後來才想要 push 兩筆資料 :-) 
    processTr(tr)  # 其實直接寫 processTr(trs[0]) 即可
    buildTable( )  # 參看前面處理 trs[1] ..
    line2 ="\r\n"
    for gg in table:    
       yy = table[gg][0]   
       if gg == "觀測時間":   # 特別處理這項
          yy = yy.strftime("%Y-%b-%d %H:%M:%S")
       line2 += gg +":" + yy
       line2 += "\r\n"
    #print("LINE2: ", line2, flush=True)   # for debug  

sendCount = 0 
def grabCWBthenPush( ): 
  global gotInput2, theInput2, allDead, driver, sendCount
  sleep(12);   # give you several time to do binding in your project
  while True:
    initCWB( );
    prepareData( );
    driver.quit()
    # print(line)
    # print(line2)
    while gotInput2:  # 不要跟別人搶 
        time.sleep(0.1)
        continue  # go back to while
    theInput2 = line
    #print("Will send LINE ONE, then sleep a while ==", flush=True)
    gotInput2 = True
    sleep(1);
    while gotInput2:  # 不要跟別人搶
        time.sleep(0.1)
        continue  # go back to while
    theInput2 = line2
    #print("Will send LINE2, then sleep a while ==", flush=True)
    gotInput2 = True  
    sleep(1.25)
    sendCount += 1
    theInput2 = "== ----- End of " + reginTitle +"--" + str(sendCount)
    gotInput2 = True  
    if(allDead) : break
    sleep(15)
    if(allDead) : break

def doRead( ): 
    global gotInput, theInput, allDead
    while True:
        while gotInput:
           time.sleep(0.1)
           continue  # go back to while
        try:
           theInput = input("Give me data: ")
        except Exception:    ##  KeyboardInterrupt:
           allDead = True
           print("\n\nDeregister " + DAN.profile['d_name'] + " !!!\n",  flush=True)
           DAN.deregister()
           sys.stdout = sys.__stdout__
           print(" Thread say Bye bye ---------------", flush=True)
           sys.exit( );   ## break  # raise   #  ?
        gotInput=True
        if theInput =='quit' or theInput == "exit": allDead = True;
        else: print("Will send " + theInput, end="   , ")

#creat a thread to do Input data from keyboard, by tsaiwn@cs.nctu.edu.tw
threadx = threading.Thread(target=doRead)
threadx.daemon = True  # 魔鬼(server) 在主thread結束後也會自殺！
threadx.start()   # 這時 threadx 開始 "同時" 執行
thready = threading.Thread(target=grabCWBthenPush)
thready.daemon = True    # daemon 的 a 不發音 == 魔鬼 == server program
thready.start()   # 這時 thready 也開始 "同時" 執行

line="line first"
line2="line2"     # the following function is used for debug
def check_alive(him):   # check a thread to see if it is alive?
    him.join(timeout=0.0)
    return him.is_alive() 
cnt = 0    ##  for debug 
gotInput = False
gotInput2 = False
while True:     ##  這是 main thread 主執行緒的　典型寫法 : 一個 Loop !
    if(allDead): break;  
    cnt += 1    ## for debug ..
    if cnt % 20 == 0:   # for debug to check thready is alive
       # print("is y-thread alive: ", check_alive(thready), end=", ")
       # print(cnt)
       pass
#  # 以上用來 check 負責抓 CWB 資料的 thread 是否還活著
    try:   ## main thread 依序做以下 (1)(2)(3) 三件事: 
    #(1)Pull data from a device feature called "Dummy_Control" 
        value1=DAN.pull('Dummy_Control')
        if value1 != None:
            print (value1[0])  
    #(2)Push keyboard data to a device feature called "Dummy_Sensor", if any 
        if gotInput:
           if theInput =='quit' or theInput=="exit":
              break;  #  sys.exit( );  
           '''
           #value2=random.uniform(1, 10)
           try:
              value2=float( theInput )
           except:
              value2=0
              ''' 
           if(allDead): break;
           #DAN.push ('Dummy_Sensor', value2,  value2)  #  試這:  
           DAN.push('Dummy_Sensor', theInput)  
           gotInput=False   # so that you can input again 
    #(3)Push CWB data from crawler to a device feature called "Dummy_Sensor", if any 
        if gotInput2:
           DAN.push('Dummy_Sensor', theInput2)
           # DAN.push('Dummy_Sensor', "543 push ggg yyy")   # for debug
           gotInput2=False    ## 通知 thready 爬蟲

    except KeyboardInterrupt:    ## 敲了 CTRL_C
        break;
    except Exception as e:    ## 不明原因, 可能剛剛網路斷了!
        print(e)
        if str(e).find('mac_addr not found:') != -1:
            print('Reg_addr is not found. Try to re-register...')
            DAN.device_registration_with_retry(ServerURL, Reg_addr)
        else:
            print('Connection failed due to unknow reasons.')
            time.sleep(1)    
    try:
       time.sleep(0.2)
    except KeyboardInterrupt:
       break
time.sleep(0.5)
try: 
   DAN.deregister()
except Exception as e:
   print("===")
print("Bye ! --------------", flush=True)
sys.exit( );   ## Quit the program