source: other-projects/maori-lang-detection/mongodb-data/7miInURLPath_exclNZ_byCountryCode.json@ 33868

Last change on this file since 33868 was 33844, checked in by ak19, 4 years ago

Regenerated

File size: 20.8 KB
Line 
1/*
2Need to inspect all those URLs with mi in URL path (mi.* or */mi) that are not sites with nz TLD or originating in NZ:
3
4db.getCollection('Websites').find({$and: [{numPagesContainingMRI: {$gt: 0}},{urlContainsLangCodeInPath: true}, {domain: {$not: /.nz$/}}, {geoLocationCountryCode: {$ne: "NZ"}}]}).count()
5472
6
7(vs:
8db.getCollection('Websites').find({$and: [{numPagesInMRI: {$gt: 0}},{urlContainsLangCodeInPath: true}, {domain: {$not: /.nz$/}}, {geoLocationCountryCode: {$ne: "NZ"}}]}).count()
9209)
10
11
12db.Websites.aggregate([
13 {
14 $match: {
15 $and: [{numPagesContainingMRI: {$gt: 0}},{urlContainsLangCodeInPath: true}, {domain: {$not: /.nz$/}}, {geoLocationCountryCode: {$ne: "NZ"}}]
16 }
17 },
18 {$group: {_id: "$geoLocationCountryCode", count: {$sum: 1}, domain: { $addToSet: '$domain' }}},
19 { $sort : { count : -1} }
20])
21
22
23But leave out Australia, as it's included in the sets for 4 and 5 that include all websites from NZ or AU OR all websites from overseas that don't have mi in URL path.
24
25db.getCollection('Websites').find({$and: [{numPagesContainingMRI: {$gt: 0}},{urlContainsLangCodeInPath: true}, {domain: {$not: /.nz$/}}, {geoLocationCountryCode: {$not: /(NZ|AU)/}}]}).count()
26471
27
28db.Websites.aggregate([
29 {
30 $match: {
31 $and: [{numPagesContainingMRI: {$gt: 0}},{urlContainsLangCodeInPath: true}, {domain: {$not: /.nz$/}}, {geoLocationCountryCode: {$not: /(NZ|AU)/}}]
32 }
33 },
34 {$group: {_id: "$geoLocationCountryCode", count: {$sum: 1}, domain: { $addToSet: '$domain' }}},
35 { $sort : { count : -1} }
36])
37
38*/
39
40/* 1 */
41{
42 "_id" : "US",
43 "count" : 305.0,
44 "domain" : [
45 "http://www.sog-pump.com",
46 "http://www.htwindsolarpower.com",
47 "http://www.ycautoc.com",
48 "http://www.restart-industry.com",
49 "https://mi.nyecountdown.com",
50 "http://www.wavesspring.com",
51 "http://www.tubemillcn.com",
52 "http://www.jindunlaobao.com",
53 "https://facebook.roseconverter.com",
54 "https://www.bestpvcfence.com",
55 "http://www.sps-squeegee.com",
56 "http://www.toption-ingredients.com",
57 "http://www.mksmartcard.com",
58 "http://mi.hongwugas.com",
59 "http://www.lishin.cc",
60 "http://infomutt.com",
61 "https://jobdescriptionsample.org",
62 "http://www.bmaxmachine.com",
63 "http://www.sdxhhd.com",
64 "http://binaryoptionsindicators.com",
65 "http://www.mao-shuo.com",
66 "http://www.eternal-friendship.com",
67 "http://www.conele-mixer.com",
68 "http://www.jhc-nonwoven.com",
69 "http://www.forever-moving.com",
70 "http://lingeriefc.com",
71 "http://indigenousblogs.com",
72 "https://www.weld-automation.com",
73 "http://www.homewin88.com",
74 "https://biblia.gospelprime.com.br",
75 "http://www.sunnymaycn.com",
76 "https://www.jlextract.com",
77 "http://www.artmetalcn.com",
78 "http://www.steel-in-china.com",
79 "http://www.nbbvc.com",
80 "https://mamaclub.info",
81 "http://www.zjnbzy.com",
82 "https://www.sdspraybooth.com",
83 "http://www.sokenswitch.com",
84 "https://follow3rs.com",
85 "http://www.autosunsoul.com",
86 "http://www.chinaocan.com",
87 "https://www.prostepper.com",
88 "https://www.yogemcasting.com",
89 "http://www.julongjewelry.cn",
90 "http://www.shengxinsport.com",
91 "http://portal.smart-project.info",
92 "http://cdn.centrallanguageschool.com",
93 "https://www.tianjia-lock.com",
94 "https://www.yourcloudlibrary.com",
95 "http://milfsplease.com",
96 "http://www.risepipe.com",
97 "http://www.pressurelantern.com",
98 "https://www.drickinstruments.com",
99 "http://www.gormeet.com",
100 "http://www.wf-fastener.com",
101 "http://www.windsolarchina.com",
102 "https://www.nbkeming.com",
103 "http://www.mtpak.com",
104 "http://www.nantaidiesel.com",
105 "https://www.judinwire.net",
106 "http://www.weldpipemill.com",
107 "http://www.ictctruss.com",
108 "http://www.analiabriz.com",
109 "http://www.xida-electronics.com",
110 "https://mi.m.wikipedia.org",
111 "https://www.junschem.com",
112 "http://www.bestwaytowhitenteethguide.org",
113 "https://www.airpullfilter.com",
114 "https://worldstarhiphop.roseconverter.com",
115 "http://www.sdtzgloves.com",
116 "http://www.teda-hydraulic.com",
117 "http://www.gmk-valve.com",
118 "https://usahello.org",
119 "https://www.datemypet.com",
120 "http://www.joyseaplywood.com",
121 "http://www.wpcline.com",
122 "https://www.hengweihoseclamp.com",
123 "https://www.td-casting.com",
124 "http://church-of-christ.org",
125 "http://www.shshenyong.com",
126 "http://www.pvcroofingtile.com",
127 "http://www.wrdtubemill.com",
128 "http://www.arjextrailerparts.com",
129 "http://www.mytrickstips.com",
130 "http://www.quickcncmachine.com",
131 "http://www.cnsongben.com",
132 "https://vk.roseconverter.com",
133 "https://www.livehoster.com",
134 "https://cycletraderpro.com",
135 "http://www.focusway-casting.com",
136 "http://www.sindadisplay.com",
137 "https://www.axnewdisplay.com",
138 "http://www.americasportsfloor.com",
139 "http://www.dmdryer.com",
140 "https://mi.centr-zashity.ru",
141 "http://csunplugged.org",
142 "https://www.inpnurseryproducts.com",
143 "https://www.hjfoodmachinery.com",
144 "http://loginmail.online",
145 "http://www.bluekin.com",
146 "http://www.hqftex.com",
147 "http://www.kehengmixing.com",
148 "https://blondewebcamgirl.com",
149 "http://www.bst-elecs.com",
150 "https://www.nickel-alloy.net",
151 "http://www.albertnovosino.com",
152 "http://www.ttyzfilter.com",
153 "https://www.risenltd.com",
154 "https://www.sinodryair.com",
155 "http://www.nicerelay.com",
156 "https://www.pldyes.com",
157 "http://www.china-brewhouse.com",
158 "http://www.ntvigourbrush.com",
159 "http://www.hobbycarbon.com",
160 "http://www.bdknitting.com",
161 "http://www.sxceramic.com",
162 "http://www.secondhormone.com",
163 "https://www.czzhit.com",
164 "http://www.zhonghe222.com",
165 "http://www.church-of-christ.org",
166 "http://www.chinacarbonfibre.com",
167 "http://www.amcbox.com",
168 "http://www.livepro-beauty.com",
169 "http://www.nbyobo.com",
170 "http://www.nyforgedwheels.com",
171 "https://guidebooq.com",
172 "http://www.sunshinebelt.com",
173 "https://www.hello4x4.com",
174 "http://www.seasum.cn",
175 "http://www.jointcontrols.net",
176 "http://www.steelprotectionpack.com",
177 "http://www.ruifeng-leather.com",
178 "http://www.suoxuehuwai.com",
179 "http://www.cnyaonan.com",
180 "http://www-hotmail-com.email",
181 "https://twitter.roseconverter.com",
182 "http://www.qitai-adhesive.com",
183 "http://www.cnfreda.com",
184 "http://www.weld-automation.com",
185 "http://www.jbpcba.com.cn",
186 "http://www.litbright-candles.com",
187 "http://www.tjcywires.com",
188 "http://www.nide-industry.com",
189 "https://www.aquagem.com.cn",
190 "http://www.ladybagcn.com",
191 "http://atoall.com",
192 "http://www.vango-tech.com",
193 "http://www.vigor-industry.com",
194 "http://www.shenhe-bearing.com",
195 "https://mi.lawyers.cafe",
196 "http://www.newbaoquan.com",
197 "https://blockchains.io",
198 "https://www.gigalight.com",
199 "http://www.fanhaopets.com",
200 "http://www.inpnurseryproducts.com",
201 "http://www.huamachinery.com",
202 "http://www.damiser.com",
203 "http://www.shanghailangzhiweld.com",
204 "http://www.yixinhetrade.com",
205 "http://www.cnfeinade.com",
206 "http://www.prostepper.com",
207 "http://www.goethe.de",
208 "https://www.njkeyuda.com",
209 "https://www.kubbamachine.com",
210 "http://www.yrkseal.com",
211 "http://www.xfinsulation.com",
212 "http://www.lanlinprintech.com",
213 "https://www.king-pcb.com",
214 "https://policies.oclc.org",
215 "http://www.ainuogas.com",
216 "http://technobuzzer.com",
217 "https://www.csunplugged.org",
218 "https://www.tkthvac.com",
219 "http://www.brushcutterjusen.com",
220 "http://www.szhaiwang.com",
221 "http://www.gemnice.com",
222 "http://www.richina-tools.com",
223 "https://www.conele-mixer.com",
224 "http://www.accotech.net",
225 "https://www.dshprecision.com",
226 "http://www.nicehut-window.com",
227 "http://www.btmeac.com",
228 "http://jobdescriptionsample.org",
229 "http://www.strongsaw.com",
230 "http://www.cannapresso.com",
231 "https://www.cz-juteng.com",
232 "http://www.aluminum-profiles-supplier.com",
233 "http://www.ledecofr.com",
234 "http://www.rosin-kings.com",
235 "http://www.tkfanen.com",
236 "http://www.sdcncrouter.com",
237 "http://www.cnxh-electric.com",
238 "http://www.twtvalvecn.com",
239 "http://www.envicool.net",
240 "http://www.jiejingfactory.com",
241 "http://www.pamaens.com",
242 "http://www.liweimetal.com",
243 "http://mi.tccasdic.com",
244 "http://www.hzhinew.com",
245 "http://www.huaxinfurnace.com",
246 "http://www.linphos.com",
247 "http://www.sinemagnetic.com",
248 "http://www.longda-inc.com",
249 "http://www.silicone-odm.com",
250 "http://www.weddingfurniture.com",
251 "http://www.touchdisplays-tech.com",
252 "http://www.houshenshoes.com",
253 "http://www.chinacombinerbox.com",
254 "http://www.xzc9.com",
255 "http://mi.broadcastbeat.com",
256 "http://www.czldfloor.com",
257 "http://www.qypaperbox.com",
258 "https://mi.wikipedia.org",
259 "https://www.everfineplastics.com",
260 "https://www.huadongmedical.com",
261 "http://www.chuamotor.com",
262 "http://www.ksdoing.com",
263 "http://www.evaescort.net",
264 "http://www.jlgrating.com",
265 "http://www.qbd-group.com",
266 "https://dwsolo.com",
267 "http://www.czzhit.com",
268 "http://www.hs-stationery.com",
269 "http://www.allutertech.com",
270 "https://www.samsungwiremesh.com",
271 "http://www.wellformpacking.com",
272 "http://www.longs-motor.com",
273 "http://www.kd-physicalrehab.com",
274 "http://www.meihua-wm.com",
275 "http://www.wzdongyi.com",
276 "http://www.pxbaisheng.com",
277 "http://www.wellfit-sportswear.com",
278 "http://www.wosaicabinet.com",
279 "http://www.chinatopcnc.com",
280 "https://drugsinc.eu",
281 "http://www.bigrollscloth.com",
282 "http://www.jiajiebathmirror.com",
283 "http://www.cz-juteng.com",
284 "http://www.shengrunqiche.com",
285 "http://www.luluae.com",
286 "http://www.jpslurrypump.com",
287 "http://www.soontruepackaging.com",
288 "https://www.judipak.com",
289 "http://www.samewe.net",
290 "http://www.infomutt.com",
291 "https://www.aquark.com.cn",
292 "http://www.tongyujiaju.com",
293 "http://www.chinapipemills.com",
294 "http://www.fxctool.com",
295 "http://mi.guoguangelectric.com",
296 "http://topbitcoincard.com",
297 "https://atoall.com",
298 "https://mi.kidspicturedictionary.com",
299 "http://www.chinabosun.com",
300 "http://www.b-packaging.com",
301 "http://www.longxin-global.com",
302 "http://www.ncpcvet.com",
303 "http://www.acouplefortheroad.com",
304 "http://www.ruk-tech.com",
305 "https://www.artiegarden.com",
306 "http://www.fxpremiere.com",
307 "https://www.tymexnetting.com",
308 "http://www.whties.com",
309 "http://blicanada.net",
310 "http://www.homey-tec.com",
311 "http://www.szebo.com",
312 "http://www.cnrgxy.com",
313 "http://www.zhenchengscrew.com",
314 "http://www.honglu-mining.com",
315 "http://www.photoprofix.com",
316 "http://www.supplyfurniture.com",
317 "http://www.linbaymachinery.com",
318 "http://mytrickstips.com",
319 "http://www.ldsolarpv.com",
320 "http://www.qxmic.com",
321 "https://www.fxcc.com",
322 "http://www.ncpcpharma.com",
323 "http://www.evergrowingcage.com",
324 "https://www.rikoooo.com",
325 "https://maxspeedtest.com",
326 "http://www.qjfiberglass.com",
327 "https://www.nbwinwinea.com",
328 "https://www.tjshenzhoutong.com",
329 "http://www.zhengmaoelec.com",
330 "http://www.chinagxmy.com",
331 "http://www.fancyco.com",
332 "http://www.tangres100.com",
333 "https://www.valve-pipe-fitting.com",
334 "http://www.gfh-electric.com",
335 "http://www.chinachairtable.com",
336 "http://www.qymachines.com",
337 "http://www.hzzjair.com",
338 "https://2fish.co",
339 "https://www.fctele.com",
340 "https://vimeo.roseconverter.com",
341 "http://www.viairdoormat.com",
342 "http://www.gecko-kalimba.com",
343 "http://mi.gmpmetalwork.com",
344 "https://www.glorystarlaser.com",
345 "http://www.wanmaroto.com",
346 "https://www.webhostingsecretrevealed.net",
347 "https://www.waterproof-factory.com",
348 "http://www.omnicnc.com"
349 ]
350}
351
352/* 2 */
353{
354 "_id" : "CN",
355 "count" : 113.0,
356 "domain" : [
357 "https://www.szradiant.com",
358 "https://www.outstandingdm.com",
359 "http://www.gmmdjx.com",
360 "http://www.likvchina.com",
361 "https://www.slagremoving.com",
362 "https://www.abdindustrial.com",
363 "https://www.c-superun.com",
364 "https://www.sino-masterbatch.com",
365 "http://www.cntiescarf.com",
366 "https://www.szhtpmart.com",
367 "https://www.dm-compressor.com",
368 "https://www.phhydraulic.com",
369 "https://www.imposalight.com",
370 "https://www.medke.com",
371 "https://www.aerial-display.com",
372 "http://www.koowheel.com",
373 "https://www.haitungchem.com",
374 "http://www.eburn-burner.com",
375 "http://www.medicohongkong.com",
376 "https://www.cntfsolar.com",
377 "https://www.diamante-tech.com",
378 "https://www.aoxinhvacr.com",
379 "https://www.richest-group.com",
380 "http://www.world-starter.com",
381 "http://www.goldenlaser.cc",
382 "https://www.safesworld.com",
383 "https://www.km-medicine.com",
384 "https://www.peptidejymed.com",
385 "https://www.nbhengchen.com",
386 "https://www.xinyuesteel.com",
387 "https://www.lasonparts.com",
388 "https://www.charmingmetal.com",
389 "http://www.abdindustrial.com",
390 "https://www.ngyc.com",
391 "https://www.pacopower.com",
392 "https://www.tjtgsteel.com",
393 "https://www.yangrutingtrade.com",
394 "http://www.wedacdisplays.com",
395 "https://www.gaofeng-petro.com",
396 "https://www.szzhsbag.com",
397 "https://www.ez-walk.com",
398 "https://www.simphoenix.com",
399 "http://www.focuslasersystems.com",
400 "http://www.kavounautoparts.com",
401 "http://www.zypackag.com",
402 "https://www.fc-med.com",
403 "https://www.bestardoors.com",
404 "https://www.foocles.com",
405 "https://www.jsjlmachinery.com",
406 "http://www.wenwencf.com",
407 "https://www.insharevape.com",
408 "https://www.n2o2gas.com",
409 "https://www.special-metal.com",
410 "https://www.dghk-buffer.com",
411 "https://www.nfyo.com",
412 "https://www.changjia-machinery.com",
413 "https://www.chinarfidcard.com",
414 "https://www.jsbotanics.com",
415 "https://www.sjzhgw.com",
416 "http://www.estarspareparts.com",
417 "https://www.zhongxinlighting.com",
418 "https://www.qdruidetai.com",
419 "https://www.study-mandarin.com",
420 "https://www.bescatray.com",
421 "http://www.qjqdvalve.com",
422 "https://www.painting-machine.com",
423 "https://www.tianseoffice.com",
424 "https://www.herbal-ingredients.com",
425 "https://www.qlart.com",
426 "https://www.sehenda-en.com",
427 "https://www.egbadges.com",
428 "http://www.3drambery.com",
429 "http://www.eudemonbaby.com",
430 "https://www.zjyongqi.com",
431 "http://www.jsbotanics.com",
432 "https://www.rswires.com",
433 "https://www.chinawelken.com",
434 "https://www.micropreparedslides.com",
435 "https://www.rykay.com",
436 "http://www.longtopmining.com",
437 "https://www.grandstarcn.com",
438 "http://www.wigglewires.com",
439 "https://www.sdtoplit.com",
440 "https://www.wecare-life.com",
441 "https://www.bailixin.com",
442 "http://www.refinehotelsupply.com",
443 "http://www.prius-automatic.com",
444 "https://www.nbulboy.com",
445 "https://www.jy-glass.com",
446 "https://www.band-ss.com",
447 "https://www.hytokstech.com",
448 "http://www.ankaicnc.com",
449 "https://www.goldnard.com",
450 "http://www.comfortebicycle.com",
451 "https://www.3drambery.com",
452 "https://www.zengrit.com",
453 "https://www.pakite.com",
454 "https://www.xianglin-plastics.com",
455 "https://www.nbjiatong.com",
456 "https://www.inductorchina.com",
457 "https://www.sakysteel.com",
458 "https://www.bofanpc.com",
459 "http://www.coneleqd.com",
460 "http://www.nbwellrun.com",
461 "https://www.jewellrylove.com",
462 "http://www.jetwayamenities.com",
463 "https://www.coffbrewing.com",
464 "https://www.aootan.com",
465 "http://www.yulong-cellulose-cmc.com",
466 "https://english.taiergroup.com",
467 "http://www.czhengfa.com",
468 "https://www.fibereye2.com",
469 "https://www.sitzonechair.com"
470 ]
471}
472
473/* 3 */
474{
475 "_id" : "FR",
476 "count" : 19.0,
477 "domain" : [
478 "https://www.casino.uk.com",
479 "https://www.planetkeyboard.com",
480 "https://mi.gem.agency",
481 "https://mi.hghphuket.com",
482 "https://mi.mehmetdursun.av.tr",
483 "https://mi.mhthread.com",
484 "https://mi.phcoker.com",
485 "https://mi.hyperbaric-chamber.com",
486 "https://mi.usa-casino-online.com",
487 "http://mi.aasraw.com",
488 "http://mi.psychicbonus.com",
489 "https://mi.apicmo.com",
490 "https://www.slotsltd.com",
491 "https://mi.isearch.de",
492 "https://mi.petrpikora.com",
493 "http://mi.fitnessrebates.com",
494 "http://www.gpedia.com",
495 "https://www.expresscasino.com",
496 "http://mi.outboard-boat-motor-repair.com"
497 ]
498}
499
500/* 4 */
501{
502 "_id" : "DE",
503 "count" : 8.0,
504 "domain" : [
505 "https://afrikhepri.org",
506 "http://transposh.org",
507 "https://mi.vessoft.com",
508 "http://www.almancax.com",
509 "https://transposh.org",
510 "https://traynews.com",
511 "https://herocity.de",
512 "https://www.saper-link-news.com"
513 ]
514}
515
516/* 5 */
517{
518 "_id" : "NL",
519 "count" : 6.0,
520 "domain" : [
521 "http://www.spectrumschool.be",
522 "http://www.martinvrijland.nl",
523 "https://www.emergency-live.com",
524 "https://realtytenerife.com",
525 "https://www.bitbybitbook.com",
526 "http://www.cbdolievoordelen.nl"
527 ]
528}
529
530/* 6 */
531{
532 "_id" : "CA",
533 "count" : 5.0,
534 "domain" : [
535 "https://www.wikiplanet.click",
536 "http://newsrule.com",
537 "http://dehaut.com",
538 "https://www.chinanbdb.com",
539 "https://cloudsfeed.com"
540 ]
541}
542
543/* 7 */
544{
545 "_id" : "GB",
546 "count" : 2.0,
547 "domain" : [
548 "https://www.centrallanguageschool.com",
549 "https://www.solasolv.com"
550 ]
551}
552
553/* 8 */
554{
555 "_id" : "HK",
556 "count" : 2.0,
557 "domain" : [
558 "https://www.desunpump.com",
559 "http://www.10turntables.com"
560 ]
561}
562
563/* 9 */
564{
565 "_id" : "UNKNOWN",
566 "count" : 2.0,
567 "domain" : [
568 "https://mi.buyaas.com",
569 "http://en.wiki.wintoflash.com"
570 ]
571}
572
573/* 10 */
574{
575 "_id" : "UA",
576 "count" : 2.0,
577 "domain" : [
578 "http://umsa.admission.center",
579 "http://ukraine.admission.center"
580 ]
581}
582
583/* 11 */
584{
585 "_id" : "SG",
586 "count" : 1.0,
587 "domain" : [
588 "https://omg-solutions.com"
589 ]
590}
591
592/* 12 */
593{
594 "_id" : "RU",
595 "count" : 1.0,
596 "domain" : [
597 "http://www.treningmozga.com"
598 ]
599}
600
601/* 13 */
602{
603 "_id" : "ES",
604 "count" : 1.0,
605 "domain" : [
606 "https://www.torresbus.es"
607 ]
608}
609
610/* 14 */
611{
612 "_id" : "IE",
613 "count" : 1.0,
614 "domain" : [
615 "http://netkiosk.co.uk"
616 ]
617}
618
619/* 15 */
620{
621 "_id" : "TR",
622 "count" : 1.0,
623 "domain" : [
624 "https://www.elitedeluxe.com.tr"
625 ]
626}
627
628/* 16 */
629{
630 "_id" : "EU",
631 "count" : 1.0,
632 "domain" : [
633 "http://www.the-good-stuff-factory.be"
634 ]
635}
636
637/* 17 */
638{
639 "_id" : "JP",
640 "count" : 1.0,
641 "domain" : [
642 "https://forexmania.org"
643 ]
644}
Note: See TracBrowser for help on using the repository browser.